ob-metaflow-stubs 6.0.5.1__py2.py3-none-any.whl → 6.0.6.0__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +927 -927
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/cli_components/__init__.pyi +2 -2
- metaflow-stubs/cli_components/utils.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +2 -2
- metaflow-stubs/meta_files.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +32 -32
- metaflow-stubs/metaflow_git.pyi +2 -2
- metaflow-stubs/mf_extensions/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +3 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +7 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +11 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +8 -9
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/ob_internal.pyi +2 -2
- metaflow-stubs/packaging_sys/__init__.pyi +6 -6
- metaflow-stubs/packaging_sys/backend.pyi +3 -3
- metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
- metaflow-stubs/packaging_sys/tar_backend.pyi +6 -6
- metaflow-stubs/packaging_sys/utils.pyi +2 -2
- metaflow-stubs/packaging_sys/v1.pyi +2 -2
- metaflow-stubs/parameters.pyi +2 -2
- metaflow-stubs/plugins/__init__.pyi +14 -14
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/argo/exit_hooks.pyi +2 -2
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +5 -5
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +5 -5
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +5 -5
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
- metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
- metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
- metaflow-stubs/plugins/secrets/utils.pyi +2 -2
- metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +32 -32
- metaflow-stubs/runner/deployer_impl.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +3 -3
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +3 -3
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/user_configs/__init__.pyi +2 -2
- metaflow-stubs/user_configs/config_options.pyi +2 -2
- metaflow-stubs/user_configs/config_parameters.pyi +6 -6
- metaflow-stubs/user_decorators/__init__.pyi +2 -2
- metaflow-stubs/user_decorators/common.pyi +2 -2
- metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
- metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
- metaflow-stubs/user_decorators/user_flow_decorator.pyi +3 -3
- metaflow-stubs/user_decorators/user_step_decorator.pyi +5 -5
- {ob_metaflow_stubs-6.0.5.1.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.6.0.dist-info/RECORD +261 -0
- ob_metaflow_stubs-6.0.5.1.dist-info/RECORD +0 -261
- {ob_metaflow_stubs-6.0.5.1.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.5.1.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.
|
4
|
-
# Generated on 2025-08-
|
3
|
+
# MF version: 2.17.0.1+obcheckpoint(0.2.4);ob(v1) #
|
4
|
+
# Generated on 2025-08-05T23:30:10.180789 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -39,8 +39,8 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
|
|
39
39
|
from .user_decorators.user_step_decorator import StepMutator as StepMutator
|
40
40
|
from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
|
41
41
|
from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
|
42
|
-
from . import cards as cards
|
43
42
|
from . import metaflow_git as metaflow_git
|
43
|
+
from . import cards as cards
|
44
44
|
from . import tuple_util as tuple_util
|
45
45
|
from . import events as events
|
46
46
|
from . import runner as runner
|
@@ -48,9 +48,9 @@ from . import plugins as plugins
|
|
48
48
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
49
49
|
from . import includefile as includefile
|
50
50
|
from .includefile import IncludeFile as IncludeFile
|
51
|
-
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
52
51
|
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
53
52
|
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
53
|
+
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
54
54
|
from . import client as client
|
55
55
|
from .client.core import namespace as namespace
|
56
56
|
from .client.core import get_namespace as get_namespace
|
@@ -168,92 +168,35 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
168
168
|
...
|
169
169
|
|
170
170
|
@typing.overload
|
171
|
-
def
|
172
|
-
"""
|
173
|
-
Specifies that the step will success under all circumstances.
|
174
|
-
|
175
|
-
The decorator will create an optional artifact, specified by `var`, which
|
176
|
-
contains the exception raised. You can use it to detect the presence
|
177
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
178
|
-
are missing.
|
179
|
-
|
180
|
-
|
181
|
-
Parameters
|
182
|
-
----------
|
183
|
-
var : str, optional, default None
|
184
|
-
Name of the artifact in which to store the caught exception.
|
185
|
-
If not specified, the exception is not stored.
|
186
|
-
print_exception : bool, default True
|
187
|
-
Determines whether or not the exception is printed to
|
188
|
-
stdout when caught.
|
189
|
-
"""
|
190
|
-
...
|
191
|
-
|
192
|
-
@typing.overload
|
193
|
-
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
194
|
-
...
|
195
|
-
|
196
|
-
@typing.overload
|
197
|
-
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
198
|
-
...
|
199
|
-
|
200
|
-
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
201
|
-
"""
|
202
|
-
Specifies that the step will success under all circumstances.
|
203
|
-
|
204
|
-
The decorator will create an optional artifact, specified by `var`, which
|
205
|
-
contains the exception raised. You can use it to detect the presence
|
206
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
207
|
-
are missing.
|
208
|
-
|
209
|
-
|
210
|
-
Parameters
|
211
|
-
----------
|
212
|
-
var : str, optional, default None
|
213
|
-
Name of the artifact in which to store the caught exception.
|
214
|
-
If not specified, the exception is not stored.
|
215
|
-
print_exception : bool, default True
|
216
|
-
Determines whether or not the exception is printed to
|
217
|
-
stdout when caught.
|
218
|
-
"""
|
219
|
-
...
|
220
|
-
|
221
|
-
@typing.overload
|
222
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
171
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
223
172
|
"""
|
224
|
-
Specifies
|
225
|
-
the execution of a step.
|
173
|
+
Specifies environment variables to be set prior to the execution of a step.
|
226
174
|
|
227
175
|
|
228
176
|
Parameters
|
229
177
|
----------
|
230
|
-
|
231
|
-
|
232
|
-
role : str, optional, default: None
|
233
|
-
Role to use for fetching secrets
|
178
|
+
vars : Dict[str, str], default {}
|
179
|
+
Dictionary of environment variables to set.
|
234
180
|
"""
|
235
181
|
...
|
236
182
|
|
237
183
|
@typing.overload
|
238
|
-
def
|
184
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
239
185
|
...
|
240
186
|
|
241
187
|
@typing.overload
|
242
|
-
def
|
188
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
243
189
|
...
|
244
190
|
|
245
|
-
def
|
191
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
246
192
|
"""
|
247
|
-
Specifies
|
248
|
-
the execution of a step.
|
193
|
+
Specifies environment variables to be set prior to the execution of a step.
|
249
194
|
|
250
195
|
|
251
196
|
Parameters
|
252
197
|
----------
|
253
|
-
|
254
|
-
|
255
|
-
role : str, optional, default: None
|
256
|
-
Role to use for fetching secrets
|
198
|
+
vars : Dict[str, str], default {}
|
199
|
+
Dictionary of environment variables to set.
|
257
200
|
"""
|
258
201
|
...
|
259
202
|
|
@@ -275,21 +218,57 @@ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepF
|
|
275
218
|
...
|
276
219
|
|
277
220
|
@typing.overload
|
278
|
-
def
|
221
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
279
222
|
"""
|
280
|
-
|
281
|
-
to
|
223
|
+
Specifies the number of times the task corresponding
|
224
|
+
to a step needs to be retried.
|
225
|
+
|
226
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
227
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
228
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
229
|
+
|
230
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
231
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
232
|
+
ensuring that the flow execution can continue.
|
233
|
+
|
234
|
+
|
235
|
+
Parameters
|
236
|
+
----------
|
237
|
+
times : int, default 3
|
238
|
+
Number of times to retry this task.
|
239
|
+
minutes_between_retries : int, default 2
|
240
|
+
Number of minutes between retries.
|
282
241
|
"""
|
283
242
|
...
|
284
243
|
|
285
244
|
@typing.overload
|
286
|
-
def
|
245
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
287
246
|
...
|
288
247
|
|
289
|
-
|
248
|
+
@typing.overload
|
249
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
250
|
+
...
|
251
|
+
|
252
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
290
253
|
"""
|
291
|
-
|
292
|
-
to
|
254
|
+
Specifies the number of times the task corresponding
|
255
|
+
to a step needs to be retried.
|
256
|
+
|
257
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
258
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
259
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
260
|
+
|
261
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
262
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
263
|
+
ensuring that the flow execution can continue.
|
264
|
+
|
265
|
+
|
266
|
+
Parameters
|
267
|
+
----------
|
268
|
+
times : int, default 3
|
269
|
+
Number of times to retry this task.
|
270
|
+
minutes_between_retries : int, default 2
|
271
|
+
Number of minutes between retries.
|
293
272
|
"""
|
294
273
|
...
|
295
274
|
|
@@ -344,278 +323,83 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
344
323
|
"""
|
345
324
|
...
|
346
325
|
|
347
|
-
|
326
|
+
@typing.overload
|
327
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
348
328
|
"""
|
349
|
-
|
350
|
-
|
351
|
-
User code call
|
352
|
-
--------------
|
353
|
-
@vllm(
|
354
|
-
model="...",
|
355
|
-
...
|
356
|
-
)
|
329
|
+
Specifies a timeout for your step.
|
357
330
|
|
358
|
-
|
359
|
-
---------------------
|
360
|
-
- 'local': Run as a separate process on the local task machine.
|
331
|
+
This decorator is useful if this step may hang indefinitely.
|
361
332
|
|
362
|
-
|
363
|
-
|
364
|
-
|
333
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
334
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
335
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
365
336
|
|
366
|
-
|
367
|
-
|
337
|
+
Note that all the values specified in parameters are added together so if you specify
|
338
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
368
339
|
|
369
340
|
|
370
341
|
Parameters
|
371
342
|
----------
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
Default is False (uses native engine).
|
379
|
-
Set to True for backward compatibility with existing code.
|
380
|
-
debug: bool
|
381
|
-
Whether to turn on verbose debugging logs.
|
382
|
-
card_refresh_interval: int
|
383
|
-
Interval in seconds for refreshing the vLLM status card.
|
384
|
-
Only used when openai_api_server=True.
|
385
|
-
max_retries: int
|
386
|
-
Maximum number of retries checking for vLLM server startup.
|
387
|
-
Only used when openai_api_server=True.
|
388
|
-
retry_alert_frequency: int
|
389
|
-
Frequency of alert logs for vLLM server startup retries.
|
390
|
-
Only used when openai_api_server=True.
|
391
|
-
engine_args : dict
|
392
|
-
Additional keyword arguments to pass to the vLLM engine.
|
393
|
-
For example, `tensor_parallel_size=2`.
|
343
|
+
seconds : int, default 0
|
344
|
+
Number of seconds to wait prior to timing out.
|
345
|
+
minutes : int, default 0
|
346
|
+
Number of minutes to wait prior to timing out.
|
347
|
+
hours : int, default 0
|
348
|
+
Number of hours to wait prior to timing out.
|
394
349
|
"""
|
395
350
|
...
|
396
351
|
|
397
352
|
@typing.overload
|
398
|
-
def
|
353
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
354
|
+
...
|
355
|
+
|
356
|
+
@typing.overload
|
357
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
358
|
+
...
|
359
|
+
|
360
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
399
361
|
"""
|
400
|
-
|
362
|
+
Specifies a timeout for your step.
|
401
363
|
|
402
|
-
|
403
|
-
- Saving Models
|
404
|
-
```python
|
405
|
-
@model
|
406
|
-
@step
|
407
|
-
def train(self):
|
408
|
-
# current.model.save returns a dictionary reference to the model saved
|
409
|
-
self.my_model = current.model.save(
|
410
|
-
path_to_my_model,
|
411
|
-
label="my_model",
|
412
|
-
metadata={
|
413
|
-
"epochs": 10,
|
414
|
-
"batch-size": 32,
|
415
|
-
"learning-rate": 0.001,
|
416
|
-
}
|
417
|
-
)
|
418
|
-
self.next(self.test)
|
364
|
+
This decorator is useful if this step may hang indefinitely.
|
419
365
|
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
# `current.model.loaded` returns a dictionary of the loaded models
|
424
|
-
# where the key is the name of the artifact and the value is the path to the model
|
425
|
-
print(os.listdir(current.model.loaded["my_model"]))
|
426
|
-
self.next(self.end)
|
427
|
-
```
|
366
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
367
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
368
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
428
369
|
|
429
|
-
|
430
|
-
|
431
|
-
@step
|
432
|
-
def train(self):
|
433
|
-
# current.model.load returns the path to the model loaded
|
434
|
-
checkpoint_path = current.model.load(
|
435
|
-
self.checkpoint_key,
|
436
|
-
)
|
437
|
-
model_path = current.model.load(
|
438
|
-
self.model,
|
439
|
-
)
|
440
|
-
self.next(self.test)
|
441
|
-
```
|
370
|
+
Note that all the values specified in parameters are added together so if you specify
|
371
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
442
372
|
|
443
373
|
|
444
374
|
Parameters
|
445
375
|
----------
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
temp_dir_root : str, default: None
|
454
|
-
The root directory under which `current.model.loaded` will store loaded models
|
376
|
+
seconds : int, default 0
|
377
|
+
Number of seconds to wait prior to timing out.
|
378
|
+
minutes : int, default 0
|
379
|
+
Number of minutes to wait prior to timing out.
|
380
|
+
hours : int, default 0
|
381
|
+
Number of hours to wait prior to timing out.
|
455
382
|
"""
|
456
383
|
...
|
457
384
|
|
458
385
|
@typing.overload
|
459
|
-
def
|
460
|
-
...
|
461
|
-
|
462
|
-
@typing.overload
|
463
|
-
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
464
|
-
...
|
465
|
-
|
466
|
-
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
386
|
+
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
467
387
|
"""
|
468
|
-
|
469
|
-
|
470
|
-
|
471
|
-
- Saving Models
|
472
|
-
```python
|
473
|
-
@model
|
474
|
-
@step
|
475
|
-
def train(self):
|
476
|
-
# current.model.save returns a dictionary reference to the model saved
|
477
|
-
self.my_model = current.model.save(
|
478
|
-
path_to_my_model,
|
479
|
-
label="my_model",
|
480
|
-
metadata={
|
481
|
-
"epochs": 10,
|
482
|
-
"batch-size": 32,
|
483
|
-
"learning-rate": 0.001,
|
484
|
-
}
|
485
|
-
)
|
486
|
-
self.next(self.test)
|
487
|
-
|
488
|
-
@model(load="my_model")
|
489
|
-
@step
|
490
|
-
def test(self):
|
491
|
-
# `current.model.loaded` returns a dictionary of the loaded models
|
492
|
-
# where the key is the name of the artifact and the value is the path to the model
|
493
|
-
print(os.listdir(current.model.loaded["my_model"]))
|
494
|
-
self.next(self.end)
|
495
|
-
```
|
496
|
-
|
497
|
-
- Loading models
|
498
|
-
```python
|
499
|
-
@step
|
500
|
-
def train(self):
|
501
|
-
# current.model.load returns the path to the model loaded
|
502
|
-
checkpoint_path = current.model.load(
|
503
|
-
self.checkpoint_key,
|
504
|
-
)
|
505
|
-
model_path = current.model.load(
|
506
|
-
self.model,
|
507
|
-
)
|
508
|
-
self.next(self.test)
|
509
|
-
```
|
510
|
-
|
511
|
-
|
512
|
-
Parameters
|
513
|
-
----------
|
514
|
-
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
515
|
-
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
516
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
517
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
518
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
519
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
520
|
-
|
521
|
-
temp_dir_root : str, default: None
|
522
|
-
The root directory under which `current.model.loaded` will store loaded models
|
388
|
+
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
389
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
390
|
+
a Neo Cloud like Nebius.
|
523
391
|
"""
|
524
392
|
...
|
525
393
|
|
526
|
-
|
527
|
-
|
528
|
-
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
529
|
-
|
530
|
-
> Examples
|
531
|
-
|
532
|
-
**Usage: creating references of models from huggingface that may be loaded in downstream steps**
|
533
|
-
```python
|
534
|
-
@huggingface_hub
|
535
|
-
@step
|
536
|
-
def pull_model_from_huggingface(self):
|
537
|
-
# `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
|
538
|
-
# and saves it in the backend storage based on the model's `repo_id`. If there exists a model
|
539
|
-
# with the same `repo_id` in the backend storage, it will not download the model again. The return
|
540
|
-
# value of the function is a reference to the model in the backend storage.
|
541
|
-
# This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
|
542
|
-
|
543
|
-
self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
|
544
|
-
self.llama_model = current.huggingface_hub.snapshot_download(
|
545
|
-
repo_id=self.model_id,
|
546
|
-
allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
|
547
|
-
)
|
548
|
-
self.next(self.train)
|
549
|
-
```
|
550
|
-
|
551
|
-
**Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
|
552
|
-
```python
|
553
|
-
@huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
|
554
|
-
@step
|
555
|
-
def pull_model_from_huggingface(self):
|
556
|
-
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
557
|
-
```
|
558
|
-
|
559
|
-
```python
|
560
|
-
@huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
|
561
|
-
@step
|
562
|
-
def finetune_model(self):
|
563
|
-
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
564
|
-
# path_to_model will be /my-directory
|
565
|
-
```
|
566
|
-
|
567
|
-
```python
|
568
|
-
# Takes all the arguments passed to `snapshot_download`
|
569
|
-
# except for `local_dir`
|
570
|
-
@huggingface_hub(load=[
|
571
|
-
{
|
572
|
-
"repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
|
573
|
-
},
|
574
|
-
{
|
575
|
-
"repo_id": "myorg/mistral-lora",
|
576
|
-
"repo_type": "model",
|
577
|
-
},
|
578
|
-
])
|
579
|
-
@step
|
580
|
-
def finetune_model(self):
|
581
|
-
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
582
|
-
# path_to_model will be /my-directory
|
583
|
-
```
|
584
|
-
|
585
|
-
|
586
|
-
Parameters
|
587
|
-
----------
|
588
|
-
temp_dir_root : str, optional
|
589
|
-
The root directory that will hold the temporary directory where objects will be downloaded.
|
590
|
-
|
591
|
-
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
592
|
-
The list of repos (models/datasets) to load.
|
593
|
-
|
594
|
-
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
595
|
-
|
596
|
-
- If repo (model/dataset) is not found in the datastore:
|
597
|
-
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
598
|
-
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
599
|
-
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
600
|
-
|
601
|
-
- If repo is found in the datastore:
|
602
|
-
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
603
|
-
"""
|
394
|
+
@typing.overload
|
395
|
+
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
604
396
|
...
|
605
397
|
|
606
|
-
def
|
398
|
+
def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
607
399
|
"""
|
608
|
-
|
609
|
-
|
610
|
-
|
611
|
-
Parameters
|
612
|
-
----------
|
613
|
-
gpu : int
|
614
|
-
Number of GPUs to use.
|
615
|
-
gpu_type : str
|
616
|
-
Type of Nvidia GPU to use.
|
617
|
-
queue_timeout : int
|
618
|
-
Time to keep the job in NVCF's queue.
|
400
|
+
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
401
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
402
|
+
a Neo Cloud like Nebius.
|
619
403
|
"""
|
620
404
|
...
|
621
405
|
|
@@ -766,58 +550,6 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
|
|
766
550
|
"""
|
767
551
|
...
|
768
552
|
|
769
|
-
@typing.overload
|
770
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
771
|
-
"""
|
772
|
-
Decorator prototype for all step decorators. This function gets specialized
|
773
|
-
and imported for all decorators types by _import_plugin_decorators().
|
774
|
-
"""
|
775
|
-
...
|
776
|
-
|
777
|
-
@typing.overload
|
778
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
779
|
-
...
|
780
|
-
|
781
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
782
|
-
"""
|
783
|
-
Decorator prototype for all step decorators. This function gets specialized
|
784
|
-
and imported for all decorators types by _import_plugin_decorators().
|
785
|
-
"""
|
786
|
-
...
|
787
|
-
|
788
|
-
@typing.overload
|
789
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
790
|
-
"""
|
791
|
-
Specifies environment variables to be set prior to the execution of a step.
|
792
|
-
|
793
|
-
|
794
|
-
Parameters
|
795
|
-
----------
|
796
|
-
vars : Dict[str, str], default {}
|
797
|
-
Dictionary of environment variables to set.
|
798
|
-
"""
|
799
|
-
...
|
800
|
-
|
801
|
-
@typing.overload
|
802
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
803
|
-
...
|
804
|
-
|
805
|
-
@typing.overload
|
806
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
807
|
-
...
|
808
|
-
|
809
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
810
|
-
"""
|
811
|
-
Specifies environment variables to be set prior to the execution of a step.
|
812
|
-
|
813
|
-
|
814
|
-
Parameters
|
815
|
-
----------
|
816
|
-
vars : Dict[str, str], default {}
|
817
|
-
Dictionary of environment variables to set.
|
818
|
-
"""
|
819
|
-
...
|
820
|
-
|
821
553
|
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
822
554
|
"""
|
823
555
|
Specifies that this step should execute on Kubernetes.
|
@@ -907,93 +639,102 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
907
639
|
"""
|
908
640
|
...
|
909
641
|
|
910
|
-
|
911
|
-
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
912
|
-
"""
|
913
|
-
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
914
|
-
It exists to make it easier for users to know that this decorator should only be used with
|
915
|
-
a Neo Cloud like CoreWeave.
|
916
|
-
"""
|
917
|
-
...
|
918
|
-
|
919
|
-
@typing.overload
|
920
|
-
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
921
|
-
...
|
922
|
-
|
923
|
-
def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
924
|
-
"""
|
925
|
-
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
926
|
-
It exists to make it easier for users to know that this decorator should only be used with
|
927
|
-
a Neo Cloud like CoreWeave.
|
928
|
-
"""
|
929
|
-
...
|
930
|
-
|
931
|
-
@typing.overload
|
932
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
642
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
933
643
|
"""
|
934
|
-
|
935
|
-
to a step needs to be retried.
|
936
|
-
|
937
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
938
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
939
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
940
|
-
|
941
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
942
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
943
|
-
ensuring that the flow execution can continue.
|
644
|
+
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
944
645
|
|
646
|
+
> Examples
|
945
647
|
|
946
|
-
|
947
|
-
|
948
|
-
|
949
|
-
|
950
|
-
|
951
|
-
|
952
|
-
|
953
|
-
|
954
|
-
|
955
|
-
|
956
|
-
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
957
|
-
...
|
958
|
-
|
959
|
-
@typing.overload
|
960
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
961
|
-
...
|
962
|
-
|
963
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
964
|
-
"""
|
965
|
-
Specifies the number of times the task corresponding
|
966
|
-
to a step needs to be retried.
|
648
|
+
**Usage: creating references of models from huggingface that may be loaded in downstream steps**
|
649
|
+
```python
|
650
|
+
@huggingface_hub
|
651
|
+
@step
|
652
|
+
def pull_model_from_huggingface(self):
|
653
|
+
# `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
|
654
|
+
# and saves it in the backend storage based on the model's `repo_id`. If there exists a model
|
655
|
+
# with the same `repo_id` in the backend storage, it will not download the model again. The return
|
656
|
+
# value of the function is a reference to the model in the backend storage.
|
657
|
+
# This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
|
967
658
|
|
968
|
-
|
969
|
-
|
970
|
-
|
659
|
+
self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
|
660
|
+
self.llama_model = current.huggingface_hub.snapshot_download(
|
661
|
+
repo_id=self.model_id,
|
662
|
+
allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
|
663
|
+
)
|
664
|
+
self.next(self.train)
|
665
|
+
```
|
971
666
|
|
972
|
-
|
973
|
-
|
974
|
-
|
667
|
+
**Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
|
668
|
+
```python
|
669
|
+
@huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
|
670
|
+
@step
|
671
|
+
def pull_model_from_huggingface(self):
|
672
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
673
|
+
```
|
674
|
+
|
675
|
+
```python
|
676
|
+
@huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
|
677
|
+
@step
|
678
|
+
def finetune_model(self):
|
679
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
680
|
+
# path_to_model will be /my-directory
|
681
|
+
```
|
682
|
+
|
683
|
+
```python
|
684
|
+
# Takes all the arguments passed to `snapshot_download`
|
685
|
+
# except for `local_dir`
|
686
|
+
@huggingface_hub(load=[
|
687
|
+
{
|
688
|
+
"repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
|
689
|
+
},
|
690
|
+
{
|
691
|
+
"repo_id": "myorg/mistral-lora",
|
692
|
+
"repo_type": "model",
|
693
|
+
},
|
694
|
+
])
|
695
|
+
@step
|
696
|
+
def finetune_model(self):
|
697
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
698
|
+
# path_to_model will be /my-directory
|
699
|
+
```
|
975
700
|
|
976
701
|
|
977
702
|
Parameters
|
978
703
|
----------
|
979
|
-
|
980
|
-
|
981
|
-
|
982
|
-
|
704
|
+
temp_dir_root : str, optional
|
705
|
+
The root directory that will hold the temporary directory where objects will be downloaded.
|
706
|
+
|
707
|
+
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
708
|
+
The list of repos (models/datasets) to load.
|
709
|
+
|
710
|
+
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
711
|
+
|
712
|
+
- If repo (model/dataset) is not found in the datastore:
|
713
|
+
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
714
|
+
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
715
|
+
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
716
|
+
|
717
|
+
- If repo is found in the datastore:
|
718
|
+
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
983
719
|
"""
|
984
720
|
...
|
985
721
|
|
986
|
-
|
722
|
+
@typing.overload
|
723
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
987
724
|
"""
|
988
|
-
|
989
|
-
|
990
|
-
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
|
995
|
-
|
996
|
-
|
725
|
+
Decorator prototype for all step decorators. This function gets specialized
|
726
|
+
and imported for all decorators types by _import_plugin_decorators().
|
727
|
+
"""
|
728
|
+
...
|
729
|
+
|
730
|
+
@typing.overload
|
731
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
732
|
+
...
|
733
|
+
|
734
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
735
|
+
"""
|
736
|
+
Decorator prototype for all step decorators. This function gets specialized
|
737
|
+
and imported for all decorators types by _import_plugin_decorators().
|
997
738
|
"""
|
998
739
|
...
|
999
740
|
|
@@ -1047,202 +788,223 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
1047
788
|
...
|
1048
789
|
|
1049
790
|
@typing.overload
|
1050
|
-
def
|
791
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1051
792
|
"""
|
1052
|
-
|
1053
|
-
|
1054
|
-
a Neo Cloud like Nebius.
|
793
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
794
|
+
to inject a card and render simple markdown content.
|
1055
795
|
"""
|
1056
796
|
...
|
1057
797
|
|
1058
798
|
@typing.overload
|
1059
|
-
def
|
799
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1060
800
|
...
|
1061
801
|
|
1062
|
-
def
|
802
|
+
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1063
803
|
"""
|
1064
|
-
|
1065
|
-
|
1066
|
-
a Neo Cloud like Nebius.
|
804
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
805
|
+
to inject a card and render simple markdown content.
|
1067
806
|
"""
|
1068
807
|
...
|
1069
808
|
|
1070
|
-
|
1071
|
-
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
809
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1072
810
|
"""
|
1073
|
-
Specifies
|
1074
|
-
|
1075
|
-
Use `@resources` to specify the resource requirements
|
1076
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1077
|
-
|
1078
|
-
You can choose the compute layer on the command line by executing e.g.
|
1079
|
-
```
|
1080
|
-
python myflow.py run --with batch
|
1081
|
-
```
|
1082
|
-
or
|
1083
|
-
```
|
1084
|
-
python myflow.py run --with kubernetes
|
1085
|
-
```
|
1086
|
-
which executes the flow on the desired system using the
|
1087
|
-
requirements specified in `@resources`.
|
811
|
+
Specifies that this step should execute on DGX cloud.
|
1088
812
|
|
1089
813
|
|
1090
814
|
Parameters
|
1091
815
|
----------
|
1092
|
-
|
1093
|
-
Number of
|
1094
|
-
|
1095
|
-
|
1096
|
-
disk : int, optional, default None
|
1097
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1098
|
-
memory : int, default 4096
|
1099
|
-
Memory size (in MB) required for this step.
|
1100
|
-
shared_memory : int, optional, default None
|
1101
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1102
|
-
This parameter maps to the `--shm-size` option in Docker.
|
816
|
+
gpu : int
|
817
|
+
Number of GPUs to use.
|
818
|
+
gpu_type : str
|
819
|
+
Type of Nvidia GPU to use.
|
1103
820
|
"""
|
1104
821
|
...
|
1105
822
|
|
1106
823
|
@typing.overload
|
1107
|
-
def
|
1108
|
-
...
|
1109
|
-
|
1110
|
-
@typing.overload
|
1111
|
-
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1112
|
-
...
|
1113
|
-
|
1114
|
-
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
824
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1115
825
|
"""
|
1116
|
-
|
826
|
+
Enables loading / saving of models within a step.
|
1117
827
|
|
1118
|
-
|
1119
|
-
|
828
|
+
> Examples
|
829
|
+
- Saving Models
|
830
|
+
```python
|
831
|
+
@model
|
832
|
+
@step
|
833
|
+
def train(self):
|
834
|
+
# current.model.save returns a dictionary reference to the model saved
|
835
|
+
self.my_model = current.model.save(
|
836
|
+
path_to_my_model,
|
837
|
+
label="my_model",
|
838
|
+
metadata={
|
839
|
+
"epochs": 10,
|
840
|
+
"batch-size": 32,
|
841
|
+
"learning-rate": 0.001,
|
842
|
+
}
|
843
|
+
)
|
844
|
+
self.next(self.test)
|
1120
845
|
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
846
|
+
@model(load="my_model")
|
847
|
+
@step
|
848
|
+
def test(self):
|
849
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
850
|
+
# where the key is the name of the artifact and the value is the path to the model
|
851
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
852
|
+
self.next(self.end)
|
1126
853
|
```
|
1127
|
-
|
854
|
+
|
855
|
+
- Loading models
|
856
|
+
```python
|
857
|
+
@step
|
858
|
+
def train(self):
|
859
|
+
# current.model.load returns the path to the model loaded
|
860
|
+
checkpoint_path = current.model.load(
|
861
|
+
self.checkpoint_key,
|
862
|
+
)
|
863
|
+
model_path = current.model.load(
|
864
|
+
self.model,
|
865
|
+
)
|
866
|
+
self.next(self.test)
|
1128
867
|
```
|
1129
|
-
which executes the flow on the desired system using the
|
1130
|
-
requirements specified in `@resources`.
|
1131
868
|
|
1132
869
|
|
1133
870
|
Parameters
|
1134
871
|
----------
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1145
|
-
This parameter maps to the `--shm-size` option in Docker.
|
872
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
873
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
874
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
875
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
876
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
877
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
878
|
+
|
879
|
+
temp_dir_root : str, default: None
|
880
|
+
The root directory under which `current.model.loaded` will store loaded models
|
1146
881
|
"""
|
1147
882
|
...
|
1148
883
|
|
1149
884
|
@typing.overload
|
1150
|
-
def
|
885
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
886
|
+
...
|
887
|
+
|
888
|
+
@typing.overload
|
889
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
890
|
+
...
|
891
|
+
|
892
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
1151
893
|
"""
|
1152
|
-
|
1153
|
-
|
1154
|
-
This decorator is useful if this step may hang indefinitely.
|
894
|
+
Enables loading / saving of models within a step.
|
1155
895
|
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
896
|
+
> Examples
|
897
|
+
- Saving Models
|
898
|
+
```python
|
899
|
+
@model
|
900
|
+
@step
|
901
|
+
def train(self):
|
902
|
+
# current.model.save returns a dictionary reference to the model saved
|
903
|
+
self.my_model = current.model.save(
|
904
|
+
path_to_my_model,
|
905
|
+
label="my_model",
|
906
|
+
metadata={
|
907
|
+
"epochs": 10,
|
908
|
+
"batch-size": 32,
|
909
|
+
"learning-rate": 0.001,
|
910
|
+
}
|
911
|
+
)
|
912
|
+
self.next(self.test)
|
1159
913
|
|
1160
|
-
|
1161
|
-
|
914
|
+
@model(load="my_model")
|
915
|
+
@step
|
916
|
+
def test(self):
|
917
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
918
|
+
# where the key is the name of the artifact and the value is the path to the model
|
919
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
920
|
+
self.next(self.end)
|
921
|
+
```
|
922
|
+
|
923
|
+
- Loading models
|
924
|
+
```python
|
925
|
+
@step
|
926
|
+
def train(self):
|
927
|
+
# current.model.load returns the path to the model loaded
|
928
|
+
checkpoint_path = current.model.load(
|
929
|
+
self.checkpoint_key,
|
930
|
+
)
|
931
|
+
model_path = current.model.load(
|
932
|
+
self.model,
|
933
|
+
)
|
934
|
+
self.next(self.test)
|
935
|
+
```
|
1162
936
|
|
1163
937
|
|
1164
938
|
Parameters
|
1165
939
|
----------
|
1166
|
-
|
1167
|
-
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
940
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
941
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
942
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
943
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
944
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
945
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
946
|
+
|
947
|
+
temp_dir_root : str, default: None
|
948
|
+
The root directory under which `current.model.loaded` will store loaded models
|
1172
949
|
"""
|
1173
950
|
...
|
1174
951
|
|
1175
|
-
|
1176
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1177
|
-
...
|
1178
|
-
|
1179
|
-
@typing.overload
|
1180
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1181
|
-
...
|
1182
|
-
|
1183
|
-
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
952
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1184
953
|
"""
|
1185
|
-
Specifies
|
1186
|
-
|
1187
|
-
This decorator is useful if this step may hang indefinitely.
|
1188
|
-
|
1189
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1190
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1191
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1192
|
-
|
1193
|
-
Note that all the values specified in parameters are added together so if you specify
|
1194
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
954
|
+
Specifies that this step should execute on DGX cloud.
|
1195
955
|
|
1196
956
|
|
1197
957
|
Parameters
|
1198
958
|
----------
|
1199
|
-
|
1200
|
-
Number of
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
"""
|
1206
|
-
...
|
1207
|
-
|
1208
|
-
@typing.overload
|
1209
|
-
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1210
|
-
"""
|
1211
|
-
Decorator prototype for all step decorators. This function gets specialized
|
1212
|
-
and imported for all decorators types by _import_plugin_decorators().
|
1213
|
-
"""
|
1214
|
-
...
|
1215
|
-
|
1216
|
-
@typing.overload
|
1217
|
-
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1218
|
-
...
|
1219
|
-
|
1220
|
-
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1221
|
-
"""
|
1222
|
-
Decorator prototype for all step decorators. This function gets specialized
|
1223
|
-
and imported for all decorators types by _import_plugin_decorators().
|
959
|
+
gpu : int
|
960
|
+
Number of GPUs to use.
|
961
|
+
gpu_type : str
|
962
|
+
Type of Nvidia GPU to use.
|
963
|
+
queue_timeout : int
|
964
|
+
Time to keep the job in NVCF's queue.
|
1224
965
|
"""
|
1225
966
|
...
|
1226
967
|
|
1227
|
-
def
|
968
|
+
def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1228
969
|
"""
|
1229
|
-
|
970
|
+
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
971
|
+
|
972
|
+
User code call
|
973
|
+
--------------
|
974
|
+
@ollama(
|
975
|
+
models=[...],
|
976
|
+
...
|
977
|
+
)
|
978
|
+
|
979
|
+
Valid backend options
|
980
|
+
---------------------
|
981
|
+
- 'local': Run as a separate process on the local task machine.
|
982
|
+
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
983
|
+
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
984
|
+
|
985
|
+
Valid model options
|
986
|
+
-------------------
|
987
|
+
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
1230
988
|
|
1231
989
|
|
1232
990
|
Parameters
|
1233
991
|
----------
|
1234
|
-
|
1235
|
-
|
1236
|
-
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1240
|
-
|
1241
|
-
|
1242
|
-
|
1243
|
-
|
1244
|
-
debug
|
1245
|
-
|
992
|
+
models: list[str]
|
993
|
+
List of Ollama containers running models in sidecars.
|
994
|
+
backend: str
|
995
|
+
Determines where and how to run the Ollama process.
|
996
|
+
force_pull: bool
|
997
|
+
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
998
|
+
cache_update_policy: str
|
999
|
+
Cache update policy: "auto", "force", or "never".
|
1000
|
+
force_cache_update: bool
|
1001
|
+
Simple override for "force" cache update policy.
|
1002
|
+
debug: bool
|
1003
|
+
Whether to turn on verbose debugging logs.
|
1004
|
+
circuit_breaker_config: dict
|
1005
|
+
Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
|
1006
|
+
timeout_config: dict
|
1007
|
+
Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
|
1246
1008
|
"""
|
1247
1009
|
...
|
1248
1010
|
|
@@ -1305,46 +1067,420 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1305
1067
|
"""
|
1306
1068
|
...
|
1307
1069
|
|
1308
|
-
|
1070
|
+
@typing.overload
|
1071
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1309
1072
|
"""
|
1310
|
-
|
1073
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1074
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1075
|
+
"""
|
1076
|
+
...
|
1077
|
+
|
1078
|
+
@typing.overload
|
1079
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1080
|
+
...
|
1081
|
+
|
1082
|
+
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1083
|
+
"""
|
1084
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1085
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1086
|
+
"""
|
1087
|
+
...
|
1088
|
+
|
1089
|
+
@typing.overload
|
1090
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1091
|
+
"""
|
1092
|
+
Specifies the resources needed when executing this step.
|
1311
1093
|
|
1312
|
-
|
1313
|
-
|
1314
|
-
@ollama(
|
1315
|
-
models=[...],
|
1316
|
-
...
|
1317
|
-
)
|
1094
|
+
Use `@resources` to specify the resource requirements
|
1095
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1318
1096
|
|
1319
|
-
|
1320
|
-
|
1321
|
-
|
1322
|
-
|
1323
|
-
|
1097
|
+
You can choose the compute layer on the command line by executing e.g.
|
1098
|
+
```
|
1099
|
+
python myflow.py run --with batch
|
1100
|
+
```
|
1101
|
+
or
|
1102
|
+
```
|
1103
|
+
python myflow.py run --with kubernetes
|
1104
|
+
```
|
1105
|
+
which executes the flow on the desired system using the
|
1106
|
+
requirements specified in `@resources`.
|
1324
1107
|
|
1325
|
-
|
1326
|
-
|
1327
|
-
|
1108
|
+
|
1109
|
+
Parameters
|
1110
|
+
----------
|
1111
|
+
cpu : int, default 1
|
1112
|
+
Number of CPUs required for this step.
|
1113
|
+
gpu : int, optional, default None
|
1114
|
+
Number of GPUs required for this step.
|
1115
|
+
disk : int, optional, default None
|
1116
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1117
|
+
memory : int, default 4096
|
1118
|
+
Memory size (in MB) required for this step.
|
1119
|
+
shared_memory : int, optional, default None
|
1120
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1121
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1122
|
+
"""
|
1123
|
+
...
|
1124
|
+
|
1125
|
+
@typing.overload
|
1126
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1127
|
+
...
|
1128
|
+
|
1129
|
+
@typing.overload
|
1130
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1131
|
+
...
|
1132
|
+
|
1133
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
1134
|
+
"""
|
1135
|
+
Specifies the resources needed when executing this step.
|
1136
|
+
|
1137
|
+
Use `@resources` to specify the resource requirements
|
1138
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1139
|
+
|
1140
|
+
You can choose the compute layer on the command line by executing e.g.
|
1141
|
+
```
|
1142
|
+
python myflow.py run --with batch
|
1143
|
+
```
|
1144
|
+
or
|
1145
|
+
```
|
1146
|
+
python myflow.py run --with kubernetes
|
1147
|
+
```
|
1148
|
+
which executes the flow on the desired system using the
|
1149
|
+
requirements specified in `@resources`.
|
1328
1150
|
|
1329
1151
|
|
1330
1152
|
Parameters
|
1331
1153
|
----------
|
1332
|
-
|
1333
|
-
|
1334
|
-
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
|
1341
|
-
|
1154
|
+
cpu : int, default 1
|
1155
|
+
Number of CPUs required for this step.
|
1156
|
+
gpu : int, optional, default None
|
1157
|
+
Number of GPUs required for this step.
|
1158
|
+
disk : int, optional, default None
|
1159
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1160
|
+
memory : int, default 4096
|
1161
|
+
Memory size (in MB) required for this step.
|
1162
|
+
shared_memory : int, optional, default None
|
1163
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1164
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1165
|
+
"""
|
1166
|
+
...
|
1167
|
+
|
1168
|
+
@typing.overload
|
1169
|
+
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1170
|
+
"""
|
1171
|
+
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
1172
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
1173
|
+
a Neo Cloud like CoreWeave.
|
1174
|
+
"""
|
1175
|
+
...
|
1176
|
+
|
1177
|
+
@typing.overload
|
1178
|
+
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1179
|
+
...
|
1180
|
+
|
1181
|
+
def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1182
|
+
"""
|
1183
|
+
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
1184
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
1185
|
+
a Neo Cloud like CoreWeave.
|
1186
|
+
"""
|
1187
|
+
...
|
1188
|
+
|
1189
|
+
@typing.overload
|
1190
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1191
|
+
"""
|
1192
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1193
|
+
the execution of a step.
|
1194
|
+
|
1195
|
+
|
1196
|
+
Parameters
|
1197
|
+
----------
|
1198
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1199
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1200
|
+
role : str, optional, default: None
|
1201
|
+
Role to use for fetching secrets
|
1202
|
+
"""
|
1203
|
+
...
|
1204
|
+
|
1205
|
+
@typing.overload
|
1206
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1207
|
+
...
|
1208
|
+
|
1209
|
+
@typing.overload
|
1210
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1211
|
+
...
|
1212
|
+
|
1213
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
1214
|
+
"""
|
1215
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1216
|
+
the execution of a step.
|
1217
|
+
|
1218
|
+
|
1219
|
+
Parameters
|
1220
|
+
----------
|
1221
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1222
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1223
|
+
role : str, optional, default: None
|
1224
|
+
Role to use for fetching secrets
|
1225
|
+
"""
|
1226
|
+
...
|
1227
|
+
|
1228
|
+
def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1229
|
+
"""
|
1230
|
+
This decorator is used to run vllm APIs as Metaflow task sidecars.
|
1231
|
+
|
1232
|
+
User code call
|
1233
|
+
--------------
|
1234
|
+
@vllm(
|
1235
|
+
model="...",
|
1236
|
+
...
|
1237
|
+
)
|
1238
|
+
|
1239
|
+
Valid backend options
|
1240
|
+
---------------------
|
1241
|
+
- 'local': Run as a separate process on the local task machine.
|
1242
|
+
|
1243
|
+
Valid model options
|
1244
|
+
-------------------
|
1245
|
+
Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
|
1246
|
+
|
1247
|
+
NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
|
1248
|
+
If you need multiple models, you must create multiple @vllm decorators.
|
1249
|
+
|
1250
|
+
|
1251
|
+
Parameters
|
1252
|
+
----------
|
1253
|
+
model: str
|
1254
|
+
HuggingFace model identifier to be served by vLLM.
|
1255
|
+
backend: str
|
1256
|
+
Determines where and how to run the vLLM process.
|
1257
|
+
openai_api_server: bool
|
1258
|
+
Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
|
1259
|
+
Default is False (uses native engine).
|
1260
|
+
Set to True for backward compatibility with existing code.
|
1342
1261
|
debug: bool
|
1343
1262
|
Whether to turn on verbose debugging logs.
|
1344
|
-
|
1345
|
-
|
1346
|
-
|
1347
|
-
|
1263
|
+
card_refresh_interval: int
|
1264
|
+
Interval in seconds for refreshing the vLLM status card.
|
1265
|
+
Only used when openai_api_server=True.
|
1266
|
+
max_retries: int
|
1267
|
+
Maximum number of retries checking for vLLM server startup.
|
1268
|
+
Only used when openai_api_server=True.
|
1269
|
+
retry_alert_frequency: int
|
1270
|
+
Frequency of alert logs for vLLM server startup retries.
|
1271
|
+
Only used when openai_api_server=True.
|
1272
|
+
engine_args : dict
|
1273
|
+
Additional keyword arguments to pass to the vLLM engine.
|
1274
|
+
For example, `tensor_parallel_size=2`.
|
1275
|
+
"""
|
1276
|
+
...
|
1277
|
+
|
1278
|
+
@typing.overload
|
1279
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1280
|
+
"""
|
1281
|
+
Specifies that the step will success under all circumstances.
|
1282
|
+
|
1283
|
+
The decorator will create an optional artifact, specified by `var`, which
|
1284
|
+
contains the exception raised. You can use it to detect the presence
|
1285
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
1286
|
+
are missing.
|
1287
|
+
|
1288
|
+
|
1289
|
+
Parameters
|
1290
|
+
----------
|
1291
|
+
var : str, optional, default None
|
1292
|
+
Name of the artifact in which to store the caught exception.
|
1293
|
+
If not specified, the exception is not stored.
|
1294
|
+
print_exception : bool, default True
|
1295
|
+
Determines whether or not the exception is printed to
|
1296
|
+
stdout when caught.
|
1297
|
+
"""
|
1298
|
+
...
|
1299
|
+
|
1300
|
+
@typing.overload
|
1301
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1302
|
+
...
|
1303
|
+
|
1304
|
+
@typing.overload
|
1305
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1306
|
+
...
|
1307
|
+
|
1308
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
1309
|
+
"""
|
1310
|
+
Specifies that the step will success under all circumstances.
|
1311
|
+
|
1312
|
+
The decorator will create an optional artifact, specified by `var`, which
|
1313
|
+
contains the exception raised. You can use it to detect the presence
|
1314
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
1315
|
+
are missing.
|
1316
|
+
|
1317
|
+
|
1318
|
+
Parameters
|
1319
|
+
----------
|
1320
|
+
var : str, optional, default None
|
1321
|
+
Name of the artifact in which to store the caught exception.
|
1322
|
+
If not specified, the exception is not stored.
|
1323
|
+
print_exception : bool, default True
|
1324
|
+
Determines whether or not the exception is printed to
|
1325
|
+
stdout when caught.
|
1326
|
+
"""
|
1327
|
+
...
|
1328
|
+
|
1329
|
+
def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1330
|
+
"""
|
1331
|
+
S3 Proxy decorator for routing S3 requests through a local proxy service.
|
1332
|
+
|
1333
|
+
|
1334
|
+
Parameters
|
1335
|
+
----------
|
1336
|
+
integration_name : str, optional
|
1337
|
+
Name of the S3 proxy integration. If not specified, will use the only
|
1338
|
+
available S3 proxy integration in the namespace (fails if multiple exist).
|
1339
|
+
write_mode : str, optional
|
1340
|
+
The desired behavior during write operations to target (origin) S3 bucket.
|
1341
|
+
allowed options are:
|
1342
|
+
"origin-and-cache" -> write to both the target S3 bucket and local object
|
1343
|
+
storage
|
1344
|
+
"origin" -> only write to the target S3 bucket
|
1345
|
+
"cache" -> only write to the object storage service used for caching
|
1346
|
+
debug : bool, optional
|
1347
|
+
Enable debug logging for proxy operations.
|
1348
|
+
"""
|
1349
|
+
...
|
1350
|
+
|
1351
|
+
@typing.overload
|
1352
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1353
|
+
"""
|
1354
|
+
Specifies the event(s) that this flow depends on.
|
1355
|
+
|
1356
|
+
```
|
1357
|
+
@trigger(event='foo')
|
1358
|
+
```
|
1359
|
+
or
|
1360
|
+
```
|
1361
|
+
@trigger(events=['foo', 'bar'])
|
1362
|
+
```
|
1363
|
+
|
1364
|
+
Additionally, you can specify the parameter mappings
|
1365
|
+
to map event payload to Metaflow parameters for the flow.
|
1366
|
+
```
|
1367
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1368
|
+
```
|
1369
|
+
or
|
1370
|
+
```
|
1371
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1372
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1373
|
+
```
|
1374
|
+
|
1375
|
+
'parameters' can also be a list of strings and tuples like so:
|
1376
|
+
```
|
1377
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1378
|
+
```
|
1379
|
+
This is equivalent to:
|
1380
|
+
```
|
1381
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1382
|
+
```
|
1383
|
+
|
1384
|
+
|
1385
|
+
Parameters
|
1386
|
+
----------
|
1387
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1388
|
+
Event dependency for this flow.
|
1389
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1390
|
+
Events dependency for this flow.
|
1391
|
+
options : Dict[str, Any], default {}
|
1392
|
+
Backend-specific configuration for tuning eventing behavior.
|
1393
|
+
"""
|
1394
|
+
...
|
1395
|
+
|
1396
|
+
@typing.overload
|
1397
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1398
|
+
...
|
1399
|
+
|
1400
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1401
|
+
"""
|
1402
|
+
Specifies the event(s) that this flow depends on.
|
1403
|
+
|
1404
|
+
```
|
1405
|
+
@trigger(event='foo')
|
1406
|
+
```
|
1407
|
+
or
|
1408
|
+
```
|
1409
|
+
@trigger(events=['foo', 'bar'])
|
1410
|
+
```
|
1411
|
+
|
1412
|
+
Additionally, you can specify the parameter mappings
|
1413
|
+
to map event payload to Metaflow parameters for the flow.
|
1414
|
+
```
|
1415
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1416
|
+
```
|
1417
|
+
or
|
1418
|
+
```
|
1419
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1420
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1421
|
+
```
|
1422
|
+
|
1423
|
+
'parameters' can also be a list of strings and tuples like so:
|
1424
|
+
```
|
1425
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1426
|
+
```
|
1427
|
+
This is equivalent to:
|
1428
|
+
```
|
1429
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1430
|
+
```
|
1431
|
+
|
1432
|
+
|
1433
|
+
Parameters
|
1434
|
+
----------
|
1435
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1436
|
+
Event dependency for this flow.
|
1437
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1438
|
+
Events dependency for this flow.
|
1439
|
+
options : Dict[str, Any], default {}
|
1440
|
+
Backend-specific configuration for tuning eventing behavior.
|
1441
|
+
"""
|
1442
|
+
...
|
1443
|
+
|
1444
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1445
|
+
"""
|
1446
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1447
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1448
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1449
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1450
|
+
starts only after all sensors finish.
|
1451
|
+
|
1452
|
+
|
1453
|
+
Parameters
|
1454
|
+
----------
|
1455
|
+
timeout : int
|
1456
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1457
|
+
poke_interval : int
|
1458
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1459
|
+
mode : str
|
1460
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1461
|
+
exponential_backoff : bool
|
1462
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1463
|
+
pool : str
|
1464
|
+
the slot pool this task should run in,
|
1465
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1466
|
+
soft_fail : bool
|
1467
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1468
|
+
name : str
|
1469
|
+
Name of the sensor on Airflow
|
1470
|
+
description : str
|
1471
|
+
Description of sensor in the Airflow UI
|
1472
|
+
bucket_key : Union[str, List[str]]
|
1473
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1474
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1475
|
+
bucket_name : str
|
1476
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1477
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1478
|
+
wildcard_match : bool
|
1479
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1480
|
+
aws_conn_id : str
|
1481
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1482
|
+
verify : bool
|
1483
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1348
1484
|
"""
|
1349
1485
|
...
|
1350
1486
|
|
@@ -1399,49 +1535,6 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
|
|
1399
1535
|
"""
|
1400
1536
|
...
|
1401
1537
|
|
1402
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1403
|
-
"""
|
1404
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1405
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1406
|
-
|
1407
|
-
|
1408
|
-
Parameters
|
1409
|
-
----------
|
1410
|
-
timeout : int
|
1411
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1412
|
-
poke_interval : int
|
1413
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1414
|
-
mode : str
|
1415
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1416
|
-
exponential_backoff : bool
|
1417
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1418
|
-
pool : str
|
1419
|
-
the slot pool this task should run in,
|
1420
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1421
|
-
soft_fail : bool
|
1422
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1423
|
-
name : str
|
1424
|
-
Name of the sensor on Airflow
|
1425
|
-
description : str
|
1426
|
-
Description of sensor in the Airflow UI
|
1427
|
-
external_dag_id : str
|
1428
|
-
The dag_id that contains the task you want to wait for.
|
1429
|
-
external_task_ids : List[str]
|
1430
|
-
The list of task_ids that you want to wait for.
|
1431
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
1432
|
-
allowed_states : List[str]
|
1433
|
-
Iterable of allowed states, (Default: ['success'])
|
1434
|
-
failed_states : List[str]
|
1435
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
1436
|
-
execution_delta : datetime.timedelta
|
1437
|
-
time difference with the previous execution to look at,
|
1438
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
1439
|
-
check_existence: bool
|
1440
|
-
Set to True to check if the external task exists or check if
|
1441
|
-
the DAG to wait for exists. (Default: True)
|
1442
|
-
"""
|
1443
|
-
...
|
1444
|
-
|
1445
1538
|
@typing.overload
|
1446
1539
|
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1447
1540
|
"""
|
@@ -1493,127 +1586,51 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
|
|
1493
1586
|
"""
|
1494
1587
|
...
|
1495
1588
|
|
1496
|
-
|
1589
|
+
@typing.overload
|
1590
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1497
1591
|
"""
|
1498
|
-
|
1499
|
-
`@checkpoint`/`@model`/`@huggingface_hub` decorators.
|
1500
|
-
|
1501
|
-
This decorator is useful when users wish to save data to a different datastore
|
1502
|
-
than what is configured in Metaflow. This can be for variety of reasons:
|
1592
|
+
Specifies the PyPI packages for all steps of the flow.
|
1503
1593
|
|
1504
|
-
|
1505
|
-
|
1506
|
-
- Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
|
1507
|
-
3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
|
1508
|
-
- Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
|
1594
|
+
Use `@pypi_base` to set common packages required by all
|
1595
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1509
1596
|
|
1510
|
-
|
1597
|
+
Parameters
|
1511
1598
|
----------
|
1599
|
+
packages : Dict[str, str], default: {}
|
1600
|
+
Packages to use for this flow. The key is the name of the package
|
1601
|
+
and the value is the version to use.
|
1602
|
+
python : str, optional, default: None
|
1603
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1604
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1605
|
+
"""
|
1606
|
+
...
|
1607
|
+
|
1608
|
+
@typing.overload
|
1609
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1610
|
+
...
|
1611
|
+
|
1612
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1613
|
+
"""
|
1614
|
+
Specifies the PyPI packages for all steps of the flow.
|
1512
1615
|
|
1513
|
-
|
1514
|
-
|
1515
|
-
```python
|
1516
|
-
@with_artifact_store(
|
1517
|
-
type="s3",
|
1518
|
-
config=lambda: {
|
1519
|
-
"root": "s3://my-bucket-foo/path/to/root",
|
1520
|
-
"role_arn": ROLE,
|
1521
|
-
},
|
1522
|
-
)
|
1523
|
-
class MyFlow(FlowSpec):
|
1524
|
-
|
1525
|
-
@checkpoint
|
1526
|
-
@step
|
1527
|
-
def start(self):
|
1528
|
-
with open("my_file.txt", "w") as f:
|
1529
|
-
f.write("Hello, World!")
|
1530
|
-
self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
|
1531
|
-
self.next(self.end)
|
1532
|
-
|
1533
|
-
```
|
1534
|
-
|
1535
|
-
- Using credentials to access the s3-compatible datastore.
|
1536
|
-
|
1537
|
-
```python
|
1538
|
-
@with_artifact_store(
|
1539
|
-
type="s3",
|
1540
|
-
config=lambda: {
|
1541
|
-
"root": "s3://my-bucket-foo/path/to/root",
|
1542
|
-
"client_params": {
|
1543
|
-
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
1544
|
-
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
1545
|
-
},
|
1546
|
-
},
|
1547
|
-
)
|
1548
|
-
class MyFlow(FlowSpec):
|
1549
|
-
|
1550
|
-
@checkpoint
|
1551
|
-
@step
|
1552
|
-
def start(self):
|
1553
|
-
with open("my_file.txt", "w") as f:
|
1554
|
-
f.write("Hello, World!")
|
1555
|
-
self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
|
1556
|
-
self.next(self.end)
|
1557
|
-
|
1558
|
-
```
|
1559
|
-
|
1560
|
-
- Accessing objects stored in external datastores after task execution.
|
1561
|
-
|
1562
|
-
```python
|
1563
|
-
run = Run("CheckpointsTestsFlow/8992")
|
1564
|
-
with artifact_store_from(run=run, config={
|
1565
|
-
"client_params": {
|
1566
|
-
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
1567
|
-
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
1568
|
-
},
|
1569
|
-
}):
|
1570
|
-
with Checkpoint() as cp:
|
1571
|
-
latest = cp.list(
|
1572
|
-
task=run["start"].task
|
1573
|
-
)[0]
|
1574
|
-
print(latest)
|
1575
|
-
cp.load(
|
1576
|
-
latest,
|
1577
|
-
"test-checkpoints"
|
1578
|
-
)
|
1616
|
+
Use `@pypi_base` to set common packages required by all
|
1617
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1579
1618
|
|
1580
|
-
|
1581
|
-
with artifact_store_from(run=run, config={
|
1582
|
-
"client_params": {
|
1583
|
-
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
1584
|
-
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
1585
|
-
},
|
1586
|
-
}):
|
1587
|
-
load_model(
|
1588
|
-
task.data.model_ref,
|
1589
|
-
"test-models"
|
1590
|
-
)
|
1591
|
-
```
|
1592
|
-
Parameters:
|
1619
|
+
Parameters
|
1593
1620
|
----------
|
1594
|
-
|
1595
|
-
|
1596
|
-
|
1597
|
-
|
1598
|
-
|
1599
|
-
|
1600
|
-
- root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
|
1601
|
-
- example: 's3://bucket-name/path/to/root'
|
1602
|
-
- example: 'gs://bucket-name/path/to/root'
|
1603
|
-
- example: 'https://myblockacc.blob.core.windows.net/metaflow/'
|
1604
|
-
- role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
|
1605
|
-
- session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
|
1606
|
-
- client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
|
1621
|
+
packages : Dict[str, str], default: {}
|
1622
|
+
Packages to use for this flow. The key is the name of the package
|
1623
|
+
and the value is the version to use.
|
1624
|
+
python : str, optional, default: None
|
1625
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1626
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1607
1627
|
"""
|
1608
1628
|
...
|
1609
1629
|
|
1610
|
-
def
|
1630
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1611
1631
|
"""
|
1612
|
-
The `@
|
1613
|
-
|
1614
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1615
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1616
|
-
starts only after all sensors finish.
|
1632
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1633
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1617
1634
|
|
1618
1635
|
|
1619
1636
|
Parameters
|
@@ -1635,18 +1652,56 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1635
1652
|
Name of the sensor on Airflow
|
1636
1653
|
description : str
|
1637
1654
|
Description of sensor in the Airflow UI
|
1638
|
-
|
1639
|
-
The
|
1640
|
-
|
1641
|
-
|
1642
|
-
|
1643
|
-
|
1644
|
-
|
1645
|
-
|
1646
|
-
|
1647
|
-
|
1648
|
-
|
1649
|
-
|
1655
|
+
external_dag_id : str
|
1656
|
+
The dag_id that contains the task you want to wait for.
|
1657
|
+
external_task_ids : List[str]
|
1658
|
+
The list of task_ids that you want to wait for.
|
1659
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1660
|
+
allowed_states : List[str]
|
1661
|
+
Iterable of allowed states, (Default: ['success'])
|
1662
|
+
failed_states : List[str]
|
1663
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1664
|
+
execution_delta : datetime.timedelta
|
1665
|
+
time difference with the previous execution to look at,
|
1666
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1667
|
+
check_existence: bool
|
1668
|
+
Set to True to check if the external task exists or check if
|
1669
|
+
the DAG to wait for exists. (Default: True)
|
1670
|
+
"""
|
1671
|
+
...
|
1672
|
+
|
1673
|
+
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1674
|
+
"""
|
1675
|
+
Specifies what flows belong to the same project.
|
1676
|
+
|
1677
|
+
A project-specific namespace is created for all flows that
|
1678
|
+
use the same `@project(name)`.
|
1679
|
+
|
1680
|
+
|
1681
|
+
Parameters
|
1682
|
+
----------
|
1683
|
+
name : str
|
1684
|
+
Project name. Make sure that the name is unique amongst all
|
1685
|
+
projects that use the same production scheduler. The name may
|
1686
|
+
contain only lowercase alphanumeric characters and underscores.
|
1687
|
+
|
1688
|
+
branch : Optional[str], default None
|
1689
|
+
The branch to use. If not specified, the branch is set to
|
1690
|
+
`user.<username>` unless `production` is set to `True`. This can
|
1691
|
+
also be set on the command line using `--branch` as a top-level option.
|
1692
|
+
It is an error to specify `branch` in the decorator and on the command line.
|
1693
|
+
|
1694
|
+
production : bool, default False
|
1695
|
+
Whether or not the branch is the production branch. This can also be set on the
|
1696
|
+
command line using `--production` as a top-level option. It is an error to specify
|
1697
|
+
`production` in the decorator and on the command line.
|
1698
|
+
The project branch name will be:
|
1699
|
+
- if `branch` is specified:
|
1700
|
+
- if `production` is True: `prod.<branch>`
|
1701
|
+
- if `production` is False: `test.<branch>`
|
1702
|
+
- if `branch` is not specified:
|
1703
|
+
- if `production` is True: `prod`
|
1704
|
+
- if `production` is False: `user.<username>`
|
1650
1705
|
"""
|
1651
1706
|
...
|
1652
1707
|
|
@@ -1751,172 +1806,117 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1751
1806
|
"""
|
1752
1807
|
...
|
1753
1808
|
|
1754
|
-
|
1755
|
-
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1809
|
+
def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
|
1756
1810
|
"""
|
1757
|
-
|
1758
|
-
|
1759
|
-
```
|
1760
|
-
@trigger(event='foo')
|
1761
|
-
```
|
1762
|
-
or
|
1763
|
-
```
|
1764
|
-
@trigger(events=['foo', 'bar'])
|
1765
|
-
```
|
1766
|
-
|
1767
|
-
Additionally, you can specify the parameter mappings
|
1768
|
-
to map event payload to Metaflow parameters for the flow.
|
1769
|
-
```
|
1770
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1771
|
-
```
|
1772
|
-
or
|
1773
|
-
```
|
1774
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1775
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1776
|
-
```
|
1811
|
+
Allows setting external datastores to save data for the
|
1812
|
+
`@checkpoint`/`@model`/`@huggingface_hub` decorators.
|
1777
1813
|
|
1778
|
-
|
1779
|
-
|
1780
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1781
|
-
```
|
1782
|
-
This is equivalent to:
|
1783
|
-
```
|
1784
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1785
|
-
```
|
1814
|
+
This decorator is useful when users wish to save data to a different datastore
|
1815
|
+
than what is configured in Metaflow. This can be for variety of reasons:
|
1786
1816
|
|
1817
|
+
1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
|
1818
|
+
2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
|
1819
|
+
- Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
|
1820
|
+
3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
|
1821
|
+
- Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
|
1787
1822
|
|
1788
|
-
|
1823
|
+
Usage:
|
1789
1824
|
----------
|
1790
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
1791
|
-
Event dependency for this flow.
|
1792
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
1793
|
-
Events dependency for this flow.
|
1794
|
-
options : Dict[str, Any], default {}
|
1795
|
-
Backend-specific configuration for tuning eventing behavior.
|
1796
|
-
"""
|
1797
|
-
...
|
1798
|
-
|
1799
|
-
@typing.overload
|
1800
|
-
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1801
|
-
...
|
1802
|
-
|
1803
|
-
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1804
|
-
"""
|
1805
|
-
Specifies the event(s) that this flow depends on.
|
1806
|
-
|
1807
|
-
```
|
1808
|
-
@trigger(event='foo')
|
1809
|
-
```
|
1810
|
-
or
|
1811
|
-
```
|
1812
|
-
@trigger(events=['foo', 'bar'])
|
1813
|
-
```
|
1814
1825
|
|
1815
|
-
|
1816
|
-
to map event payload to Metaflow parameters for the flow.
|
1817
|
-
```
|
1818
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1819
|
-
```
|
1820
|
-
or
|
1821
|
-
```
|
1822
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1823
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1824
|
-
```
|
1826
|
+
- Using a custom IAM role to access the datastore.
|
1825
1827
|
|
1826
|
-
|
1827
|
-
|
1828
|
-
|
1829
|
-
|
1830
|
-
|
1831
|
-
|
1832
|
-
|
1833
|
-
|
1828
|
+
```python
|
1829
|
+
@with_artifact_store(
|
1830
|
+
type="s3",
|
1831
|
+
config=lambda: {
|
1832
|
+
"root": "s3://my-bucket-foo/path/to/root",
|
1833
|
+
"role_arn": ROLE,
|
1834
|
+
},
|
1835
|
+
)
|
1836
|
+
class MyFlow(FlowSpec):
|
1834
1837
|
|
1838
|
+
@checkpoint
|
1839
|
+
@step
|
1840
|
+
def start(self):
|
1841
|
+
with open("my_file.txt", "w") as f:
|
1842
|
+
f.write("Hello, World!")
|
1843
|
+
self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
|
1844
|
+
self.next(self.end)
|
1835
1845
|
|
1836
|
-
|
1837
|
-
----------
|
1838
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
1839
|
-
Event dependency for this flow.
|
1840
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
1841
|
-
Events dependency for this flow.
|
1842
|
-
options : Dict[str, Any], default {}
|
1843
|
-
Backend-specific configuration for tuning eventing behavior.
|
1844
|
-
"""
|
1845
|
-
...
|
1846
|
-
|
1847
|
-
@typing.overload
|
1848
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1849
|
-
"""
|
1850
|
-
Specifies the PyPI packages for all steps of the flow.
|
1846
|
+
```
|
1851
1847
|
|
1852
|
-
|
1853
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1848
|
+
- Using credentials to access the s3-compatible datastore.
|
1854
1849
|
|
1855
|
-
|
1856
|
-
|
1857
|
-
|
1858
|
-
|
1859
|
-
|
1860
|
-
|
1861
|
-
|
1862
|
-
|
1863
|
-
|
1864
|
-
|
1865
|
-
|
1866
|
-
|
1867
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1868
|
-
...
|
1869
|
-
|
1870
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1871
|
-
"""
|
1872
|
-
Specifies the PyPI packages for all steps of the flow.
|
1850
|
+
```python
|
1851
|
+
@with_artifact_store(
|
1852
|
+
type="s3",
|
1853
|
+
config=lambda: {
|
1854
|
+
"root": "s3://my-bucket-foo/path/to/root",
|
1855
|
+
"client_params": {
|
1856
|
+
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
1857
|
+
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
1858
|
+
},
|
1859
|
+
},
|
1860
|
+
)
|
1861
|
+
class MyFlow(FlowSpec):
|
1873
1862
|
|
1874
|
-
|
1875
|
-
|
1863
|
+
@checkpoint
|
1864
|
+
@step
|
1865
|
+
def start(self):
|
1866
|
+
with open("my_file.txt", "w") as f:
|
1867
|
+
f.write("Hello, World!")
|
1868
|
+
self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
|
1869
|
+
self.next(self.end)
|
1876
1870
|
|
1877
|
-
|
1878
|
-
----------
|
1879
|
-
packages : Dict[str, str], default: {}
|
1880
|
-
Packages to use for this flow. The key is the name of the package
|
1881
|
-
and the value is the version to use.
|
1882
|
-
python : str, optional, default: None
|
1883
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1884
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1885
|
-
"""
|
1886
|
-
...
|
1887
|
-
|
1888
|
-
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1889
|
-
"""
|
1890
|
-
Specifies what flows belong to the same project.
|
1871
|
+
```
|
1891
1872
|
|
1892
|
-
|
1893
|
-
use the same `@project(name)`.
|
1873
|
+
- Accessing objects stored in external datastores after task execution.
|
1894
1874
|
|
1875
|
+
```python
|
1876
|
+
run = Run("CheckpointsTestsFlow/8992")
|
1877
|
+
with artifact_store_from(run=run, config={
|
1878
|
+
"client_params": {
|
1879
|
+
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
1880
|
+
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
1881
|
+
},
|
1882
|
+
}):
|
1883
|
+
with Checkpoint() as cp:
|
1884
|
+
latest = cp.list(
|
1885
|
+
task=run["start"].task
|
1886
|
+
)[0]
|
1887
|
+
print(latest)
|
1888
|
+
cp.load(
|
1889
|
+
latest,
|
1890
|
+
"test-checkpoints"
|
1891
|
+
)
|
1895
1892
|
|
1896
|
-
|
1893
|
+
task = Task("TorchTuneFlow/8484/train/53673")
|
1894
|
+
with artifact_store_from(run=run, config={
|
1895
|
+
"client_params": {
|
1896
|
+
"aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
|
1897
|
+
"aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
|
1898
|
+
},
|
1899
|
+
}):
|
1900
|
+
load_model(
|
1901
|
+
task.data.model_ref,
|
1902
|
+
"test-models"
|
1903
|
+
)
|
1904
|
+
```
|
1905
|
+
Parameters:
|
1897
1906
|
----------
|
1898
|
-
name : str
|
1899
|
-
Project name. Make sure that the name is unique amongst all
|
1900
|
-
projects that use the same production scheduler. The name may
|
1901
|
-
contain only lowercase alphanumeric characters and underscores.
|
1902
1907
|
|
1903
|
-
|
1904
|
-
The
|
1905
|
-
`user.<username>` unless `production` is set to `True`. This can
|
1906
|
-
also be set on the command line using `--branch` as a top-level option.
|
1907
|
-
It is an error to specify `branch` in the decorator and on the command line.
|
1908
|
+
type: str
|
1909
|
+
The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
|
1908
1910
|
|
1909
|
-
|
1910
|
-
|
1911
|
-
|
1912
|
-
|
1913
|
-
|
1914
|
-
|
1915
|
-
|
1916
|
-
|
1917
|
-
|
1918
|
-
- if `production` is True: `prod`
|
1919
|
-
- if `production` is False: `user.<username>`
|
1911
|
+
config: dict or Callable
|
1912
|
+
Dictionary of configuration options for the datastore. The following keys are required:
|
1913
|
+
- root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
|
1914
|
+
- example: 's3://bucket-name/path/to/root'
|
1915
|
+
- example: 'gs://bucket-name/path/to/root'
|
1916
|
+
- example: 'https://myblockacc.blob.core.windows.net/metaflow/'
|
1917
|
+
- role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
|
1918
|
+
- session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
|
1919
|
+
- client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
|
1920
1920
|
"""
|
1921
1921
|
...
|
1922
1922
|
|