ob-metaflow-stubs 6.0.9.2__py2.py3-none-any.whl → 6.0.9.3__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ob-metaflow-stubs might be problematic. Click here for more details.
- metaflow-stubs/__init__.pyi +812 -812
- metaflow-stubs/cards.pyi +1 -1
- metaflow-stubs/cli.pyi +1 -1
- metaflow-stubs/cli_components/__init__.pyi +1 -1
- metaflow-stubs/cli_components/utils.pyi +1 -1
- metaflow-stubs/client/__init__.pyi +1 -1
- metaflow-stubs/client/core.pyi +4 -4
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/events.pyi +1 -1
- metaflow-stubs/exception.pyi +1 -1
- metaflow-stubs/flowspec.pyi +3 -3
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +3 -3
- metaflow-stubs/meta_files.pyi +1 -1
- metaflow-stubs/metadata_provider/__init__.pyi +1 -1
- metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
- metaflow-stubs/metadata_provider/metadata.pyi +1 -1
- metaflow-stubs/metadata_provider/util.pyi +1 -1
- metaflow-stubs/metaflow_config.pyi +1 -1
- metaflow-stubs/metaflow_current.pyi +53 -53
- metaflow-stubs/metaflow_git.pyi +1 -1
- metaflow-stubs/mf_extensions/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +4 -4
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +3 -3
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +1 -1
- metaflow-stubs/multicore_utils.pyi +1 -1
- metaflow-stubs/ob_internal.pyi +1 -1
- metaflow-stubs/packaging_sys/__init__.pyi +5 -5
- metaflow-stubs/packaging_sys/backend.pyi +2 -2
- metaflow-stubs/packaging_sys/distribution_support.pyi +3 -3
- metaflow-stubs/packaging_sys/tar_backend.pyi +3 -3
- metaflow-stubs/packaging_sys/utils.pyi +1 -1
- metaflow-stubs/packaging_sys/v1.pyi +2 -2
- metaflow-stubs/parameters.pyi +2 -2
- metaflow-stubs/plugins/__init__.pyi +12 -12
- metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
- metaflow-stubs/plugins/airflow/exception.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
- metaflow-stubs/plugins/argo/__init__.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +1 -1
- metaflow-stubs/plugins/argo/exit_hooks.pyi +1 -1
- metaflow-stubs/plugins/aws/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +1 -1
- metaflow-stubs/plugins/azure/__init__.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
- metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/cards/__init__.pyi +5 -5
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
- metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
- metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
- metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
- metaflow-stubs/plugins/cards/exception.pyi +1 -1
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/local.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
- metaflow-stubs/plugins/debug_logger.pyi +1 -1
- metaflow-stubs/plugins/debug_monitor.pyi +1 -1
- metaflow-stubs/plugins/environment_decorator.pyi +1 -1
- metaflow-stubs/plugins/events_decorator.pyi +1 -1
- metaflow-stubs/plugins/exit_hook/__init__.pyi +1 -1
- metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +1 -1
- metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
- metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
- metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
- metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
- metaflow-stubs/plugins/ollama/__init__.pyi +1 -1
- metaflow-stubs/plugins/optuna/__init__.pyi +1 -1
- metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
- metaflow-stubs/plugins/perimeters.pyi +1 -1
- metaflow-stubs/plugins/project_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
- metaflow-stubs/plugins/pypi/utils.pyi +1 -1
- metaflow-stubs/plugins/resources_decorator.pyi +1 -1
- metaflow-stubs/plugins/retry_decorator.pyi +1 -1
- metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
- metaflow-stubs/plugins/secrets/secrets_func.pyi +1 -1
- metaflow-stubs/plugins/secrets/secrets_spec.pyi +1 -1
- metaflow-stubs/plugins/secrets/utils.pyi +1 -1
- metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/plugins/storage_executor.pyi +1 -1
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
- metaflow-stubs/plugins/uv/__init__.pyi +1 -1
- metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +1 -1
- metaflow-stubs/pylint_wrapper.pyi +1 -1
- metaflow-stubs/runner/__init__.pyi +1 -1
- metaflow-stubs/runner/deployer.pyi +4 -4
- metaflow-stubs/runner/deployer_impl.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +2 -2
- metaflow-stubs/runner/nbdeploy.pyi +1 -1
- metaflow-stubs/runner/nbrun.pyi +1 -1
- metaflow-stubs/runner/subprocess_manager.pyi +1 -1
- metaflow-stubs/runner/utils.pyi +3 -3
- metaflow-stubs/system/__init__.pyi +1 -1
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +1 -1
- metaflow-stubs/tagging_util.pyi +1 -1
- metaflow-stubs/tuple_util.pyi +1 -1
- metaflow-stubs/user_configs/__init__.pyi +1 -1
- metaflow-stubs/user_configs/config_options.pyi +1 -1
- metaflow-stubs/user_configs/config_parameters.pyi +4 -4
- metaflow-stubs/user_decorators/__init__.pyi +1 -1
- metaflow-stubs/user_decorators/common.pyi +1 -1
- metaflow-stubs/user_decorators/mutable_flow.pyi +3 -3
- metaflow-stubs/user_decorators/mutable_step.pyi +4 -4
- metaflow-stubs/user_decorators/user_flow_decorator.pyi +3 -3
- metaflow-stubs/user_decorators/user_step_decorator.pyi +3 -3
- {ob_metaflow_stubs-6.0.9.2.dist-info → ob_metaflow_stubs-6.0.9.3.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.9.3.dist-info/RECORD +262 -0
- ob_metaflow_stubs-6.0.9.2.dist-info/RECORD +0 -262
- {ob_metaflow_stubs-6.0.9.2.dist-info → ob_metaflow_stubs-6.0.9.3.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.9.2.dist-info → ob_metaflow_stubs-6.0.9.3.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
######################################################################################################
|
|
2
2
|
# Auto-generated Metaflow stub file #
|
|
3
3
|
# MF version: 2.18.1.1+obcheckpoint(0.2.4);ob(v1) #
|
|
4
|
-
# Generated on 2025-
|
|
4
|
+
# Generated on 2025-09-02T19:19:25.341768 #
|
|
5
5
|
######################################################################################################
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
@@ -40,9 +40,9 @@ from .user_decorators.user_step_decorator import StepMutator as StepMutator
|
|
|
40
40
|
from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
|
|
41
41
|
from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
|
|
42
42
|
from . import metaflow_git as metaflow_git
|
|
43
|
-
from . import tuple_util as tuple_util
|
|
44
|
-
from . import cards as cards
|
|
45
43
|
from . import events as events
|
|
44
|
+
from . import cards as cards
|
|
45
|
+
from . import tuple_util as tuple_util
|
|
46
46
|
from . import runner as runner
|
|
47
47
|
from . import plugins as plugins
|
|
48
48
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
|
@@ -167,57 +167,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
|
167
167
|
"""
|
|
168
168
|
...
|
|
169
169
|
|
|
170
|
-
@typing.overload
|
|
171
|
-
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
172
|
-
"""
|
|
173
|
-
Specifies that the step will success under all circumstances.
|
|
174
|
-
|
|
175
|
-
The decorator will create an optional artifact, specified by `var`, which
|
|
176
|
-
contains the exception raised. You can use it to detect the presence
|
|
177
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
|
178
|
-
are missing.
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
Parameters
|
|
182
|
-
----------
|
|
183
|
-
var : str, optional, default None
|
|
184
|
-
Name of the artifact in which to store the caught exception.
|
|
185
|
-
If not specified, the exception is not stored.
|
|
186
|
-
print_exception : bool, default True
|
|
187
|
-
Determines whether or not the exception is printed to
|
|
188
|
-
stdout when caught.
|
|
189
|
-
"""
|
|
190
|
-
...
|
|
191
|
-
|
|
192
|
-
@typing.overload
|
|
193
|
-
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
194
|
-
...
|
|
195
|
-
|
|
196
|
-
@typing.overload
|
|
197
|
-
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
198
|
-
...
|
|
199
|
-
|
|
200
|
-
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
|
201
|
-
"""
|
|
202
|
-
Specifies that the step will success under all circumstances.
|
|
203
|
-
|
|
204
|
-
The decorator will create an optional artifact, specified by `var`, which
|
|
205
|
-
contains the exception raised. You can use it to detect the presence
|
|
206
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
|
207
|
-
are missing.
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
Parameters
|
|
211
|
-
----------
|
|
212
|
-
var : str, optional, default None
|
|
213
|
-
Name of the artifact in which to store the caught exception.
|
|
214
|
-
If not specified, the exception is not stored.
|
|
215
|
-
print_exception : bool, default True
|
|
216
|
-
Determines whether or not the exception is printed to
|
|
217
|
-
stdout when caught.
|
|
218
|
-
"""
|
|
219
|
-
...
|
|
220
|
-
|
|
221
170
|
@typing.overload
|
|
222
171
|
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
223
172
|
"""
|
|
@@ -278,234 +227,178 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
|
278
227
|
...
|
|
279
228
|
|
|
280
229
|
@typing.overload
|
|
281
|
-
def
|
|
230
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
282
231
|
"""
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
> Examples
|
|
286
|
-
- Saving Models
|
|
287
|
-
```python
|
|
288
|
-
@model
|
|
289
|
-
@step
|
|
290
|
-
def train(self):
|
|
291
|
-
# current.model.save returns a dictionary reference to the model saved
|
|
292
|
-
self.my_model = current.model.save(
|
|
293
|
-
path_to_my_model,
|
|
294
|
-
label="my_model",
|
|
295
|
-
metadata={
|
|
296
|
-
"epochs": 10,
|
|
297
|
-
"batch-size": 32,
|
|
298
|
-
"learning-rate": 0.001,
|
|
299
|
-
}
|
|
300
|
-
)
|
|
301
|
-
self.next(self.test)
|
|
302
|
-
|
|
303
|
-
@model(load="my_model")
|
|
304
|
-
@step
|
|
305
|
-
def test(self):
|
|
306
|
-
# `current.model.loaded` returns a dictionary of the loaded models
|
|
307
|
-
# where the key is the name of the artifact and the value is the path to the model
|
|
308
|
-
print(os.listdir(current.model.loaded["my_model"]))
|
|
309
|
-
self.next(self.end)
|
|
310
|
-
```
|
|
311
|
-
|
|
312
|
-
- Loading models
|
|
313
|
-
```python
|
|
314
|
-
@step
|
|
315
|
-
def train(self):
|
|
316
|
-
# current.model.load returns the path to the model loaded
|
|
317
|
-
checkpoint_path = current.model.load(
|
|
318
|
-
self.checkpoint_key,
|
|
319
|
-
)
|
|
320
|
-
model_path = current.model.load(
|
|
321
|
-
self.model,
|
|
322
|
-
)
|
|
323
|
-
self.next(self.test)
|
|
324
|
-
```
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
Parameters
|
|
328
|
-
----------
|
|
329
|
-
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
|
330
|
-
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
|
331
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
|
332
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
|
333
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
|
334
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
|
335
|
-
|
|
336
|
-
temp_dir_root : str, default: None
|
|
337
|
-
The root directory under which `current.model.loaded` will store loaded models
|
|
232
|
+
Internal decorator to support Fast bakery
|
|
338
233
|
"""
|
|
339
234
|
...
|
|
340
235
|
|
|
341
236
|
@typing.overload
|
|
342
|
-
def
|
|
237
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
343
238
|
...
|
|
344
239
|
|
|
345
|
-
|
|
346
|
-
|
|
240
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
241
|
+
"""
|
|
242
|
+
Internal decorator to support Fast bakery
|
|
243
|
+
"""
|
|
347
244
|
...
|
|
348
245
|
|
|
349
|
-
|
|
246
|
+
@typing.overload
|
|
247
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
350
248
|
"""
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
> Examples
|
|
354
|
-
- Saving Models
|
|
355
|
-
```python
|
|
356
|
-
@model
|
|
357
|
-
@step
|
|
358
|
-
def train(self):
|
|
359
|
-
# current.model.save returns a dictionary reference to the model saved
|
|
360
|
-
self.my_model = current.model.save(
|
|
361
|
-
path_to_my_model,
|
|
362
|
-
label="my_model",
|
|
363
|
-
metadata={
|
|
364
|
-
"epochs": 10,
|
|
365
|
-
"batch-size": 32,
|
|
366
|
-
"learning-rate": 0.001,
|
|
367
|
-
}
|
|
368
|
-
)
|
|
369
|
-
self.next(self.test)
|
|
370
|
-
|
|
371
|
-
@model(load="my_model")
|
|
372
|
-
@step
|
|
373
|
-
def test(self):
|
|
374
|
-
# `current.model.loaded` returns a dictionary of the loaded models
|
|
375
|
-
# where the key is the name of the artifact and the value is the path to the model
|
|
376
|
-
print(os.listdir(current.model.loaded["my_model"]))
|
|
377
|
-
self.next(self.end)
|
|
378
|
-
```
|
|
249
|
+
Specifies that the step will success under all circumstances.
|
|
379
250
|
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
# current.model.load returns the path to the model loaded
|
|
385
|
-
checkpoint_path = current.model.load(
|
|
386
|
-
self.checkpoint_key,
|
|
387
|
-
)
|
|
388
|
-
model_path = current.model.load(
|
|
389
|
-
self.model,
|
|
390
|
-
)
|
|
391
|
-
self.next(self.test)
|
|
392
|
-
```
|
|
251
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
252
|
+
contains the exception raised. You can use it to detect the presence
|
|
253
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
254
|
+
are missing.
|
|
393
255
|
|
|
394
256
|
|
|
395
257
|
Parameters
|
|
396
258
|
----------
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
temp_dir_root : str, default: None
|
|
405
|
-
The root directory under which `current.model.loaded` will store loaded models
|
|
259
|
+
var : str, optional, default None
|
|
260
|
+
Name of the artifact in which to store the caught exception.
|
|
261
|
+
If not specified, the exception is not stored.
|
|
262
|
+
print_exception : bool, default True
|
|
263
|
+
Determines whether or not the exception is printed to
|
|
264
|
+
stdout when caught.
|
|
406
265
|
"""
|
|
407
266
|
...
|
|
408
267
|
|
|
409
|
-
|
|
268
|
+
@typing.overload
|
|
269
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
270
|
+
...
|
|
271
|
+
|
|
272
|
+
@typing.overload
|
|
273
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
274
|
+
...
|
|
275
|
+
|
|
276
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
|
410
277
|
"""
|
|
411
|
-
|
|
278
|
+
Specifies that the step will success under all circumstances.
|
|
279
|
+
|
|
280
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
281
|
+
contains the exception raised. You can use it to detect the presence
|
|
282
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
283
|
+
are missing.
|
|
412
284
|
|
|
413
285
|
|
|
414
286
|
Parameters
|
|
415
287
|
----------
|
|
416
|
-
|
|
417
|
-
Name of the
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
"origin-and-cache" -> write to both the target S3 bucket and local object
|
|
423
|
-
storage
|
|
424
|
-
"origin" -> only write to the target S3 bucket
|
|
425
|
-
"cache" -> only write to the object storage service used for caching
|
|
426
|
-
debug : bool, optional
|
|
427
|
-
Enable debug logging for proxy operations.
|
|
288
|
+
var : str, optional, default None
|
|
289
|
+
Name of the artifact in which to store the caught exception.
|
|
290
|
+
If not specified, the exception is not stored.
|
|
291
|
+
print_exception : bool, default True
|
|
292
|
+
Determines whether or not the exception is printed to
|
|
293
|
+
stdout when caught.
|
|
428
294
|
"""
|
|
429
295
|
...
|
|
430
296
|
|
|
431
297
|
@typing.overload
|
|
432
|
-
def
|
|
298
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
433
299
|
"""
|
|
434
|
-
|
|
435
|
-
|
|
300
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
301
|
+
|
|
302
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
436
303
|
|
|
437
304
|
|
|
438
305
|
Parameters
|
|
439
306
|
----------
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
307
|
+
type : str, default 'default'
|
|
308
|
+
Card type.
|
|
309
|
+
id : str, optional, default None
|
|
310
|
+
If multiple cards are present, use this id to identify this card.
|
|
311
|
+
options : Dict[str, Any], default {}
|
|
312
|
+
Options passed to the card. The contents depend on the card type.
|
|
313
|
+
timeout : int, default 45
|
|
314
|
+
Interrupt reporting if it takes more than this many seconds.
|
|
444
315
|
"""
|
|
445
316
|
...
|
|
446
317
|
|
|
447
318
|
@typing.overload
|
|
448
|
-
def
|
|
319
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
449
320
|
...
|
|
450
321
|
|
|
451
322
|
@typing.overload
|
|
452
|
-
def
|
|
323
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
453
324
|
...
|
|
454
325
|
|
|
455
|
-
def
|
|
326
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
|
456
327
|
"""
|
|
457
|
-
|
|
458
|
-
|
|
328
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
329
|
+
|
|
330
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
459
331
|
|
|
460
332
|
|
|
461
333
|
Parameters
|
|
462
334
|
----------
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
472
|
-
"""
|
|
473
|
-
A simple decorator that demonstrates using CardDecoratorInjector
|
|
474
|
-
to inject a card and render simple markdown content.
|
|
335
|
+
type : str, default 'default'
|
|
336
|
+
Card type.
|
|
337
|
+
id : str, optional, default None
|
|
338
|
+
If multiple cards are present, use this id to identify this card.
|
|
339
|
+
options : Dict[str, Any], default {}
|
|
340
|
+
Options passed to the card. The contents depend on the card type.
|
|
341
|
+
timeout : int, default 45
|
|
342
|
+
Interrupt reporting if it takes more than this many seconds.
|
|
475
343
|
"""
|
|
476
344
|
...
|
|
477
345
|
|
|
478
346
|
@typing.overload
|
|
479
|
-
def
|
|
480
|
-
...
|
|
481
|
-
|
|
482
|
-
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
347
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
483
348
|
"""
|
|
484
|
-
|
|
485
|
-
to
|
|
349
|
+
Specifies the number of times the task corresponding
|
|
350
|
+
to a step needs to be retried.
|
|
351
|
+
|
|
352
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
|
353
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
354
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
|
355
|
+
|
|
356
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
357
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
|
358
|
+
ensuring that the flow execution can continue.
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
Parameters
|
|
362
|
+
----------
|
|
363
|
+
times : int, default 3
|
|
364
|
+
Number of times to retry this task.
|
|
365
|
+
minutes_between_retries : int, default 2
|
|
366
|
+
Number of minutes between retries.
|
|
486
367
|
"""
|
|
487
368
|
...
|
|
488
369
|
|
|
489
370
|
@typing.overload
|
|
490
|
-
def
|
|
491
|
-
"""
|
|
492
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
493
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
494
|
-
"""
|
|
371
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
495
372
|
...
|
|
496
373
|
|
|
497
374
|
@typing.overload
|
|
498
|
-
def
|
|
375
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
499
376
|
...
|
|
500
377
|
|
|
501
|
-
def
|
|
378
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
|
502
379
|
"""
|
|
503
|
-
|
|
504
|
-
|
|
380
|
+
Specifies the number of times the task corresponding
|
|
381
|
+
to a step needs to be retried.
|
|
382
|
+
|
|
383
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
|
384
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
385
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
|
386
|
+
|
|
387
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
388
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
|
389
|
+
ensuring that the flow execution can continue.
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
Parameters
|
|
393
|
+
----------
|
|
394
|
+
times : int, default 3
|
|
395
|
+
Number of times to retry this task.
|
|
396
|
+
minutes_between_retries : int, default 2
|
|
397
|
+
Number of minutes between retries.
|
|
505
398
|
"""
|
|
506
399
|
...
|
|
507
400
|
|
|
508
|
-
def
|
|
401
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
509
402
|
"""
|
|
510
403
|
Specifies that this step should execute on DGX cloud.
|
|
511
404
|
|
|
@@ -516,92 +409,156 @@ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Cal
|
|
|
516
409
|
Number of GPUs to use.
|
|
517
410
|
gpu_type : str
|
|
518
411
|
Type of Nvidia GPU to use.
|
|
412
|
+
queue_timeout : int
|
|
413
|
+
Time to keep the job in NVCF's queue.
|
|
519
414
|
"""
|
|
520
415
|
...
|
|
521
416
|
|
|
522
417
|
@typing.overload
|
|
523
|
-
def
|
|
418
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
524
419
|
"""
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
420
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
528
421
|
|
|
529
422
|
|
|
530
423
|
Parameters
|
|
531
424
|
----------
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
id : str, optional, default None
|
|
535
|
-
If multiple cards are present, use this id to identify this card.
|
|
536
|
-
options : Dict[str, Any], default {}
|
|
537
|
-
Options passed to the card. The contents depend on the card type.
|
|
538
|
-
timeout : int, default 45
|
|
539
|
-
Interrupt reporting if it takes more than this many seconds.
|
|
425
|
+
vars : Dict[str, str], default {}
|
|
426
|
+
Dictionary of environment variables to set.
|
|
540
427
|
"""
|
|
541
428
|
...
|
|
542
429
|
|
|
543
430
|
@typing.overload
|
|
544
|
-
def
|
|
431
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
545
432
|
...
|
|
546
433
|
|
|
547
434
|
@typing.overload
|
|
548
|
-
def
|
|
435
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
549
436
|
...
|
|
550
437
|
|
|
551
|
-
def
|
|
438
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
|
552
439
|
"""
|
|
553
|
-
|
|
440
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
554
441
|
|
|
555
|
-
|
|
442
|
+
|
|
443
|
+
Parameters
|
|
444
|
+
----------
|
|
445
|
+
vars : Dict[str, str], default {}
|
|
446
|
+
Dictionary of environment variables to set.
|
|
447
|
+
"""
|
|
448
|
+
...
|
|
449
|
+
|
|
450
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
451
|
+
"""
|
|
452
|
+
Specifies that this step should execute on DGX cloud.
|
|
556
453
|
|
|
557
454
|
|
|
558
455
|
Parameters
|
|
559
456
|
----------
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
options : Dict[str, Any], default {}
|
|
565
|
-
Options passed to the card. The contents depend on the card type.
|
|
566
|
-
timeout : int, default 45
|
|
567
|
-
Interrupt reporting if it takes more than this many seconds.
|
|
457
|
+
gpu : int
|
|
458
|
+
Number of GPUs to use.
|
|
459
|
+
gpu_type : str
|
|
460
|
+
Type of Nvidia GPU to use.
|
|
568
461
|
"""
|
|
569
462
|
...
|
|
570
463
|
|
|
571
464
|
@typing.overload
|
|
572
|
-
def
|
|
465
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
573
466
|
"""
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
467
|
+
Specifies the resources needed when executing this step.
|
|
468
|
+
|
|
469
|
+
Use `@resources` to specify the resource requirements
|
|
470
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
|
471
|
+
|
|
472
|
+
You can choose the compute layer on the command line by executing e.g.
|
|
473
|
+
```
|
|
474
|
+
python myflow.py run --with batch
|
|
475
|
+
```
|
|
476
|
+
or
|
|
477
|
+
```
|
|
478
|
+
python myflow.py run --with kubernetes
|
|
479
|
+
```
|
|
480
|
+
which executes the flow on the desired system using the
|
|
481
|
+
requirements specified in `@resources`.
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
Parameters
|
|
485
|
+
----------
|
|
486
|
+
cpu : int, default 1
|
|
487
|
+
Number of CPUs required for this step.
|
|
488
|
+
gpu : int, optional, default None
|
|
489
|
+
Number of GPUs required for this step.
|
|
490
|
+
disk : int, optional, default None
|
|
491
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
|
492
|
+
memory : int, default 4096
|
|
493
|
+
Memory size (in MB) required for this step.
|
|
494
|
+
shared_memory : int, optional, default None
|
|
495
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
|
496
|
+
This parameter maps to the `--shm-size` option in Docker.
|
|
577
497
|
"""
|
|
578
498
|
...
|
|
579
499
|
|
|
580
500
|
@typing.overload
|
|
581
|
-
def
|
|
501
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
582
502
|
...
|
|
583
503
|
|
|
584
|
-
|
|
504
|
+
@typing.overload
|
|
505
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
506
|
+
...
|
|
507
|
+
|
|
508
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
|
585
509
|
"""
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
510
|
+
Specifies the resources needed when executing this step.
|
|
511
|
+
|
|
512
|
+
Use `@resources` to specify the resource requirements
|
|
513
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
|
514
|
+
|
|
515
|
+
You can choose the compute layer on the command line by executing e.g.
|
|
516
|
+
```
|
|
517
|
+
python myflow.py run --with batch
|
|
518
|
+
```
|
|
519
|
+
or
|
|
520
|
+
```
|
|
521
|
+
python myflow.py run --with kubernetes
|
|
522
|
+
```
|
|
523
|
+
which executes the flow on the desired system using the
|
|
524
|
+
requirements specified in `@resources`.
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
Parameters
|
|
528
|
+
----------
|
|
529
|
+
cpu : int, default 1
|
|
530
|
+
Number of CPUs required for this step.
|
|
531
|
+
gpu : int, optional, default None
|
|
532
|
+
Number of GPUs required for this step.
|
|
533
|
+
disk : int, optional, default None
|
|
534
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
|
535
|
+
memory : int, default 4096
|
|
536
|
+
Memory size (in MB) required for this step.
|
|
537
|
+
shared_memory : int, optional, default None
|
|
538
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
|
539
|
+
This parameter maps to the `--shm-size` option in Docker.
|
|
589
540
|
"""
|
|
590
541
|
...
|
|
591
542
|
|
|
592
|
-
def
|
|
543
|
+
def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
593
544
|
"""
|
|
594
|
-
|
|
545
|
+
S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
595
546
|
|
|
596
547
|
|
|
597
548
|
Parameters
|
|
598
549
|
----------
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
550
|
+
integration_name : str, optional
|
|
551
|
+
Name of the S3 proxy integration. If not specified, will use the only
|
|
552
|
+
available S3 proxy integration in the namespace (fails if multiple exist).
|
|
553
|
+
write_mode : str, optional
|
|
554
|
+
The desired behavior during write operations to target (origin) S3 bucket.
|
|
555
|
+
allowed options are:
|
|
556
|
+
"origin-and-cache" -> write to both the target S3 bucket and local object
|
|
557
|
+
storage
|
|
558
|
+
"origin" -> only write to the target S3 bucket
|
|
559
|
+
"cache" -> only write to the object storage service used for caching
|
|
560
|
+
debug : bool, optional
|
|
561
|
+
Enable debug logging for proxy operations.
|
|
605
562
|
"""
|
|
606
563
|
...
|
|
607
564
|
|
|
@@ -753,126 +710,61 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
|
|
|
753
710
|
...
|
|
754
711
|
|
|
755
712
|
@typing.overload
|
|
756
|
-
def
|
|
713
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
757
714
|
"""
|
|
758
|
-
Specifies
|
|
715
|
+
Specifies a timeout for your step.
|
|
759
716
|
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
717
|
+
This decorator is useful if this step may hang indefinitely.
|
|
718
|
+
|
|
719
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
720
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
721
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
722
|
+
|
|
723
|
+
Note that all the values specified in parameters are added together so if you specify
|
|
724
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
|
764
725
|
|
|
765
726
|
|
|
766
727
|
Parameters
|
|
767
728
|
----------
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
729
|
+
seconds : int, default 0
|
|
730
|
+
Number of seconds to wait prior to timing out.
|
|
731
|
+
minutes : int, default 0
|
|
732
|
+
Number of minutes to wait prior to timing out.
|
|
733
|
+
hours : int, default 0
|
|
734
|
+
Number of hours to wait prior to timing out.
|
|
774
735
|
"""
|
|
775
736
|
...
|
|
776
737
|
|
|
777
738
|
@typing.overload
|
|
778
|
-
def
|
|
739
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
779
740
|
...
|
|
780
741
|
|
|
781
742
|
@typing.overload
|
|
782
|
-
def
|
|
743
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
783
744
|
...
|
|
784
745
|
|
|
785
|
-
def
|
|
746
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
|
786
747
|
"""
|
|
787
|
-
Specifies
|
|
748
|
+
Specifies a timeout for your step.
|
|
788
749
|
|
|
789
|
-
|
|
790
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
791
|
-
you can use `@pypi_base` to set packages required by all
|
|
792
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
Parameters
|
|
796
|
-
----------
|
|
797
|
-
packages : Dict[str, str], default: {}
|
|
798
|
-
Packages to use for this step. The key is the name of the package
|
|
799
|
-
and the value is the version to use.
|
|
800
|
-
python : str, optional, default: None
|
|
801
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
802
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
803
|
-
"""
|
|
804
|
-
...
|
|
805
|
-
|
|
806
|
-
@typing.overload
|
|
807
|
-
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
808
|
-
"""
|
|
809
|
-
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
810
|
-
It exists to make it easier for users to know that this decorator should only be used with
|
|
811
|
-
a Neo Cloud like Nebius.
|
|
812
|
-
"""
|
|
813
|
-
...
|
|
814
|
-
|
|
815
|
-
@typing.overload
|
|
816
|
-
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
817
|
-
...
|
|
818
|
-
|
|
819
|
-
def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
820
|
-
"""
|
|
821
|
-
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
822
|
-
It exists to make it easier for users to know that this decorator should only be used with
|
|
823
|
-
a Neo Cloud like Nebius.
|
|
824
|
-
"""
|
|
825
|
-
...
|
|
826
|
-
|
|
827
|
-
@typing.overload
|
|
828
|
-
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
829
|
-
"""
|
|
830
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
831
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
832
|
-
"""
|
|
833
|
-
...
|
|
834
|
-
|
|
835
|
-
@typing.overload
|
|
836
|
-
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
837
|
-
...
|
|
838
|
-
|
|
839
|
-
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
840
|
-
"""
|
|
841
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
842
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
843
|
-
"""
|
|
844
|
-
...
|
|
845
|
-
|
|
846
|
-
@typing.overload
|
|
847
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
848
|
-
"""
|
|
849
|
-
Specifies environment variables to be set prior to the execution of a step.
|
|
750
|
+
This decorator is useful if this step may hang indefinitely.
|
|
850
751
|
|
|
752
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
753
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
754
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
851
755
|
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
vars : Dict[str, str], default {}
|
|
855
|
-
Dictionary of environment variables to set.
|
|
856
|
-
"""
|
|
857
|
-
...
|
|
858
|
-
|
|
859
|
-
@typing.overload
|
|
860
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
861
|
-
...
|
|
862
|
-
|
|
863
|
-
@typing.overload
|
|
864
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
865
|
-
...
|
|
866
|
-
|
|
867
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
|
868
|
-
"""
|
|
869
|
-
Specifies environment variables to be set prior to the execution of a step.
|
|
756
|
+
Note that all the values specified in parameters are added together so if you specify
|
|
757
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
|
870
758
|
|
|
871
759
|
|
|
872
760
|
Parameters
|
|
873
761
|
----------
|
|
874
|
-
|
|
875
|
-
|
|
762
|
+
seconds : int, default 0
|
|
763
|
+
Number of seconds to wait prior to timing out.
|
|
764
|
+
minutes : int, default 0
|
|
765
|
+
Number of minutes to wait prior to timing out.
|
|
766
|
+
hours : int, default 0
|
|
767
|
+
Number of hours to wait prior to timing out.
|
|
876
768
|
"""
|
|
877
769
|
...
|
|
878
770
|
|
|
@@ -920,99 +812,21 @@ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy:
|
|
|
920
812
|
...
|
|
921
813
|
|
|
922
814
|
@typing.overload
|
|
923
|
-
def
|
|
815
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
924
816
|
"""
|
|
925
|
-
|
|
817
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
818
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
926
819
|
"""
|
|
927
820
|
...
|
|
928
821
|
|
|
929
822
|
@typing.overload
|
|
930
|
-
def
|
|
931
|
-
...
|
|
932
|
-
|
|
933
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
934
|
-
"""
|
|
935
|
-
Internal decorator to support Fast bakery
|
|
936
|
-
"""
|
|
823
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
937
824
|
...
|
|
938
825
|
|
|
939
|
-
def
|
|
826
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
940
827
|
"""
|
|
941
|
-
Decorator
|
|
942
|
-
|
|
943
|
-
> Examples
|
|
944
|
-
|
|
945
|
-
**Usage: creating references of models from huggingface that may be loaded in downstream steps**
|
|
946
|
-
```python
|
|
947
|
-
@huggingface_hub
|
|
948
|
-
@step
|
|
949
|
-
def pull_model_from_huggingface(self):
|
|
950
|
-
# `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
|
|
951
|
-
# and saves it in the backend storage based on the model's `repo_id`. If there exists a model
|
|
952
|
-
# with the same `repo_id` in the backend storage, it will not download the model again. The return
|
|
953
|
-
# value of the function is a reference to the model in the backend storage.
|
|
954
|
-
# This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
|
|
955
|
-
|
|
956
|
-
self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
|
|
957
|
-
self.llama_model = current.huggingface_hub.snapshot_download(
|
|
958
|
-
repo_id=self.model_id,
|
|
959
|
-
allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
|
|
960
|
-
)
|
|
961
|
-
self.next(self.train)
|
|
962
|
-
```
|
|
963
|
-
|
|
964
|
-
**Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
|
|
965
|
-
```python
|
|
966
|
-
@huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
|
|
967
|
-
@step
|
|
968
|
-
def pull_model_from_huggingface(self):
|
|
969
|
-
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
970
|
-
```
|
|
971
|
-
|
|
972
|
-
```python
|
|
973
|
-
@huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
|
|
974
|
-
@step
|
|
975
|
-
def finetune_model(self):
|
|
976
|
-
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
977
|
-
# path_to_model will be /my-directory
|
|
978
|
-
```
|
|
979
|
-
|
|
980
|
-
```python
|
|
981
|
-
# Takes all the arguments passed to `snapshot_download`
|
|
982
|
-
# except for `local_dir`
|
|
983
|
-
@huggingface_hub(load=[
|
|
984
|
-
{
|
|
985
|
-
"repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
|
|
986
|
-
},
|
|
987
|
-
{
|
|
988
|
-
"repo_id": "myorg/mistral-lora",
|
|
989
|
-
"repo_type": "model",
|
|
990
|
-
},
|
|
991
|
-
])
|
|
992
|
-
@step
|
|
993
|
-
def finetune_model(self):
|
|
994
|
-
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
995
|
-
# path_to_model will be /my-directory
|
|
996
|
-
```
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
Parameters
|
|
1000
|
-
----------
|
|
1001
|
-
temp_dir_root : str, optional
|
|
1002
|
-
The root directory that will hold the temporary directory where objects will be downloaded.
|
|
1003
|
-
|
|
1004
|
-
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
|
1005
|
-
The list of repos (models/datasets) to load.
|
|
1006
|
-
|
|
1007
|
-
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
|
1008
|
-
|
|
1009
|
-
- If repo (model/dataset) is not found in the datastore:
|
|
1010
|
-
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
|
1011
|
-
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
|
1012
|
-
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
|
1013
|
-
|
|
1014
|
-
- If repo is found in the datastore:
|
|
1015
|
-
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
|
828
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
829
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
1016
830
|
"""
|
|
1017
831
|
...
|
|
1018
832
|
|
|
@@ -1105,196 +919,382 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
|
1105
919
|
"""
|
|
1106
920
|
...
|
|
1107
921
|
|
|
1108
|
-
|
|
1109
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
922
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1110
923
|
"""
|
|
1111
|
-
|
|
1112
|
-
to a step needs to be retried.
|
|
1113
|
-
|
|
1114
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
|
1115
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
1116
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
|
924
|
+
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
|
1117
925
|
|
|
1118
|
-
|
|
1119
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
|
1120
|
-
ensuring that the flow execution can continue.
|
|
926
|
+
> Examples
|
|
1121
927
|
|
|
928
|
+
**Usage: creating references of models from huggingface that may be loaded in downstream steps**
|
|
929
|
+
```python
|
|
930
|
+
@huggingface_hub
|
|
931
|
+
@step
|
|
932
|
+
def pull_model_from_huggingface(self):
|
|
933
|
+
# `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
|
|
934
|
+
# and saves it in the backend storage based on the model's `repo_id`. If there exists a model
|
|
935
|
+
# with the same `repo_id` in the backend storage, it will not download the model again. The return
|
|
936
|
+
# value of the function is a reference to the model in the backend storage.
|
|
937
|
+
# This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
|
|
1122
938
|
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
...
|
|
1131
|
-
|
|
1132
|
-
@typing.overload
|
|
1133
|
-
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1134
|
-
...
|
|
1135
|
-
|
|
1136
|
-
@typing.overload
|
|
1137
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1138
|
-
...
|
|
1139
|
-
|
|
1140
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
|
1141
|
-
"""
|
|
1142
|
-
Specifies the number of times the task corresponding
|
|
1143
|
-
to a step needs to be retried.
|
|
939
|
+
self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
|
|
940
|
+
self.llama_model = current.huggingface_hub.snapshot_download(
|
|
941
|
+
repo_id=self.model_id,
|
|
942
|
+
allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
|
|
943
|
+
)
|
|
944
|
+
self.next(self.train)
|
|
945
|
+
```
|
|
1144
946
|
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
947
|
+
**Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
|
|
948
|
+
```python
|
|
949
|
+
@huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
|
|
950
|
+
@step
|
|
951
|
+
def pull_model_from_huggingface(self):
|
|
952
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
953
|
+
```
|
|
1148
954
|
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
955
|
+
```python
|
|
956
|
+
@huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
|
|
957
|
+
@step
|
|
958
|
+
def finetune_model(self):
|
|
959
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
960
|
+
# path_to_model will be /my-directory
|
|
961
|
+
```
|
|
962
|
+
|
|
963
|
+
```python
|
|
964
|
+
# Takes all the arguments passed to `snapshot_download`
|
|
965
|
+
# except for `local_dir`
|
|
966
|
+
@huggingface_hub(load=[
|
|
967
|
+
{
|
|
968
|
+
"repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
|
|
969
|
+
},
|
|
970
|
+
{
|
|
971
|
+
"repo_id": "myorg/mistral-lora",
|
|
972
|
+
"repo_type": "model",
|
|
973
|
+
},
|
|
974
|
+
])
|
|
975
|
+
@step
|
|
976
|
+
def finetune_model(self):
|
|
977
|
+
path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
|
|
978
|
+
# path_to_model will be /my-directory
|
|
979
|
+
```
|
|
1152
980
|
|
|
1153
981
|
|
|
1154
982
|
Parameters
|
|
1155
983
|
----------
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
984
|
+
temp_dir_root : str, optional
|
|
985
|
+
The root directory that will hold the temporary directory where objects will be downloaded.
|
|
986
|
+
|
|
987
|
+
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
|
988
|
+
The list of repos (models/datasets) to load.
|
|
989
|
+
|
|
990
|
+
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
|
991
|
+
|
|
992
|
+
- If repo (model/dataset) is not found in the datastore:
|
|
993
|
+
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
|
994
|
+
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
|
995
|
+
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
|
996
|
+
|
|
997
|
+
- If repo is found in the datastore:
|
|
998
|
+
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
|
1160
999
|
"""
|
|
1161
1000
|
...
|
|
1162
1001
|
|
|
1163
1002
|
@typing.overload
|
|
1164
|
-
def
|
|
1003
|
+
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1165
1004
|
"""
|
|
1166
|
-
|
|
1005
|
+
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
1006
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
1007
|
+
a Neo Cloud like CoreWeave.
|
|
1008
|
+
"""
|
|
1009
|
+
...
|
|
1010
|
+
|
|
1011
|
+
@typing.overload
|
|
1012
|
+
def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1013
|
+
...
|
|
1014
|
+
|
|
1015
|
+
def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1016
|
+
"""
|
|
1017
|
+
CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
1018
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
1019
|
+
a Neo Cloud like CoreWeave.
|
|
1020
|
+
"""
|
|
1021
|
+
...
|
|
1022
|
+
|
|
1023
|
+
@typing.overload
|
|
1024
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1025
|
+
"""
|
|
1026
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
1027
|
+
the execution of a step.
|
|
1167
1028
|
|
|
1168
|
-
Use `@resources` to specify the resource requirements
|
|
1169
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
|
1170
1029
|
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1030
|
+
Parameters
|
|
1031
|
+
----------
|
|
1032
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
1033
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
1034
|
+
role : str, optional, default: None
|
|
1035
|
+
Role to use for fetching secrets
|
|
1036
|
+
"""
|
|
1037
|
+
...
|
|
1038
|
+
|
|
1039
|
+
@typing.overload
|
|
1040
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1041
|
+
...
|
|
1042
|
+
|
|
1043
|
+
@typing.overload
|
|
1044
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1045
|
+
...
|
|
1046
|
+
|
|
1047
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
|
1048
|
+
"""
|
|
1049
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
1050
|
+
the execution of a step.
|
|
1051
|
+
|
|
1052
|
+
|
|
1053
|
+
Parameters
|
|
1054
|
+
----------
|
|
1055
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
1056
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
1057
|
+
role : str, optional, default: None
|
|
1058
|
+
Role to use for fetching secrets
|
|
1059
|
+
"""
|
|
1060
|
+
...
|
|
1061
|
+
|
|
1062
|
+
@typing.overload
|
|
1063
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1064
|
+
"""
|
|
1065
|
+
Enables loading / saving of models within a step.
|
|
1066
|
+
|
|
1067
|
+
> Examples
|
|
1068
|
+
- Saving Models
|
|
1069
|
+
```python
|
|
1070
|
+
@model
|
|
1071
|
+
@step
|
|
1072
|
+
def train(self):
|
|
1073
|
+
# current.model.save returns a dictionary reference to the model saved
|
|
1074
|
+
self.my_model = current.model.save(
|
|
1075
|
+
path_to_my_model,
|
|
1076
|
+
label="my_model",
|
|
1077
|
+
metadata={
|
|
1078
|
+
"epochs": 10,
|
|
1079
|
+
"batch-size": 32,
|
|
1080
|
+
"learning-rate": 0.001,
|
|
1081
|
+
}
|
|
1082
|
+
)
|
|
1083
|
+
self.next(self.test)
|
|
1084
|
+
|
|
1085
|
+
@model(load="my_model")
|
|
1086
|
+
@step
|
|
1087
|
+
def test(self):
|
|
1088
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
|
1089
|
+
# where the key is the name of the artifact and the value is the path to the model
|
|
1090
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
|
1091
|
+
self.next(self.end)
|
|
1176
1092
|
```
|
|
1177
|
-
|
|
1093
|
+
|
|
1094
|
+
- Loading models
|
|
1095
|
+
```python
|
|
1096
|
+
@step
|
|
1097
|
+
def train(self):
|
|
1098
|
+
# current.model.load returns the path to the model loaded
|
|
1099
|
+
checkpoint_path = current.model.load(
|
|
1100
|
+
self.checkpoint_key,
|
|
1101
|
+
)
|
|
1102
|
+
model_path = current.model.load(
|
|
1103
|
+
self.model,
|
|
1104
|
+
)
|
|
1105
|
+
self.next(self.test)
|
|
1178
1106
|
```
|
|
1179
|
-
which executes the flow on the desired system using the
|
|
1180
|
-
requirements specified in `@resources`.
|
|
1181
1107
|
|
|
1182
1108
|
|
|
1183
1109
|
Parameters
|
|
1184
1110
|
----------
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
|
1195
|
-
This parameter maps to the `--shm-size` option in Docker.
|
|
1111
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
|
1112
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
|
1113
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
|
1114
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
|
1115
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
|
1116
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
|
1117
|
+
|
|
1118
|
+
temp_dir_root : str, default: None
|
|
1119
|
+
The root directory under which `current.model.loaded` will store loaded models
|
|
1196
1120
|
"""
|
|
1197
1121
|
...
|
|
1198
1122
|
|
|
1199
1123
|
@typing.overload
|
|
1200
|
-
def
|
|
1124
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1201
1125
|
...
|
|
1202
1126
|
|
|
1203
1127
|
@typing.overload
|
|
1204
|
-
def
|
|
1128
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1205
1129
|
...
|
|
1206
1130
|
|
|
1207
|
-
def
|
|
1131
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
|
1208
1132
|
"""
|
|
1209
|
-
|
|
1133
|
+
Enables loading / saving of models within a step.
|
|
1210
1134
|
|
|
1211
|
-
|
|
1212
|
-
|
|
1135
|
+
> Examples
|
|
1136
|
+
- Saving Models
|
|
1137
|
+
```python
|
|
1138
|
+
@model
|
|
1139
|
+
@step
|
|
1140
|
+
def train(self):
|
|
1141
|
+
# current.model.save returns a dictionary reference to the model saved
|
|
1142
|
+
self.my_model = current.model.save(
|
|
1143
|
+
path_to_my_model,
|
|
1144
|
+
label="my_model",
|
|
1145
|
+
metadata={
|
|
1146
|
+
"epochs": 10,
|
|
1147
|
+
"batch-size": 32,
|
|
1148
|
+
"learning-rate": 0.001,
|
|
1149
|
+
}
|
|
1150
|
+
)
|
|
1151
|
+
self.next(self.test)
|
|
1213
1152
|
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1153
|
+
@model(load="my_model")
|
|
1154
|
+
@step
|
|
1155
|
+
def test(self):
|
|
1156
|
+
# `current.model.loaded` returns a dictionary of the loaded models
|
|
1157
|
+
# where the key is the name of the artifact and the value is the path to the model
|
|
1158
|
+
print(os.listdir(current.model.loaded["my_model"]))
|
|
1159
|
+
self.next(self.end)
|
|
1219
1160
|
```
|
|
1220
|
-
|
|
1161
|
+
|
|
1162
|
+
- Loading models
|
|
1163
|
+
```python
|
|
1164
|
+
@step
|
|
1165
|
+
def train(self):
|
|
1166
|
+
# current.model.load returns the path to the model loaded
|
|
1167
|
+
checkpoint_path = current.model.load(
|
|
1168
|
+
self.checkpoint_key,
|
|
1169
|
+
)
|
|
1170
|
+
model_path = current.model.load(
|
|
1171
|
+
self.model,
|
|
1172
|
+
)
|
|
1173
|
+
self.next(self.test)
|
|
1221
1174
|
```
|
|
1222
|
-
which executes the flow on the desired system using the
|
|
1223
|
-
requirements specified in `@resources`.
|
|
1224
1175
|
|
|
1225
1176
|
|
|
1226
1177
|
Parameters
|
|
1227
1178
|
----------
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
|
1238
|
-
This parameter maps to the `--shm-size` option in Docker.
|
|
1179
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
|
1180
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
|
1181
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
|
|
1182
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
|
1183
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
|
1184
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
|
1185
|
+
|
|
1186
|
+
temp_dir_root : str, default: None
|
|
1187
|
+
The root directory under which `current.model.loaded` will store loaded models
|
|
1239
1188
|
"""
|
|
1240
1189
|
...
|
|
1241
1190
|
|
|
1242
1191
|
@typing.overload
|
|
1243
|
-
def
|
|
1192
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1244
1193
|
"""
|
|
1245
|
-
|
|
1194
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
1195
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
1196
|
+
"""
|
|
1197
|
+
...
|
|
1198
|
+
|
|
1199
|
+
@typing.overload
|
|
1200
|
+
def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1201
|
+
...
|
|
1202
|
+
|
|
1203
|
+
def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1204
|
+
"""
|
|
1205
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
1206
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
1207
|
+
"""
|
|
1208
|
+
...
|
|
1209
|
+
|
|
1210
|
+
@typing.overload
|
|
1211
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
1212
|
+
"""
|
|
1213
|
+
Specifies the PyPI packages for the step.
|
|
1246
1214
|
|
|
1247
|
-
|
|
1215
|
+
Information in this decorator will augment any
|
|
1216
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
1217
|
+
you can use `@pypi_base` to set packages required by all
|
|
1218
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1248
1219
|
|
|
1249
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
1250
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
1251
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
1252
1220
|
|
|
1253
|
-
|
|
1254
|
-
|
|
1221
|
+
Parameters
|
|
1222
|
+
----------
|
|
1223
|
+
packages : Dict[str, str], default: {}
|
|
1224
|
+
Packages to use for this step. The key is the name of the package
|
|
1225
|
+
and the value is the version to use.
|
|
1226
|
+
python : str, optional, default: None
|
|
1227
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1228
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1229
|
+
"""
|
|
1230
|
+
...
|
|
1231
|
+
|
|
1232
|
+
@typing.overload
|
|
1233
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1234
|
+
...
|
|
1235
|
+
|
|
1236
|
+
@typing.overload
|
|
1237
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1238
|
+
...
|
|
1239
|
+
|
|
1240
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
1241
|
+
"""
|
|
1242
|
+
Specifies the PyPI packages for the step.
|
|
1243
|
+
|
|
1244
|
+
Information in this decorator will augment any
|
|
1245
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
1246
|
+
you can use `@pypi_base` to set packages required by all
|
|
1247
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1255
1248
|
|
|
1256
1249
|
|
|
1257
1250
|
Parameters
|
|
1258
1251
|
----------
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1252
|
+
packages : Dict[str, str], default: {}
|
|
1253
|
+
Packages to use for this step. The key is the name of the package
|
|
1254
|
+
and the value is the version to use.
|
|
1255
|
+
python : str, optional, default: None
|
|
1256
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1257
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1258
|
+
"""
|
|
1259
|
+
...
|
|
1260
|
+
|
|
1261
|
+
@typing.overload
|
|
1262
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1263
|
+
"""
|
|
1264
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
|
1265
|
+
to inject a card and render simple markdown content.
|
|
1266
|
+
"""
|
|
1267
|
+
...
|
|
1268
|
+
|
|
1269
|
+
@typing.overload
|
|
1270
|
+
def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1271
|
+
...
|
|
1272
|
+
|
|
1273
|
+
def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1274
|
+
"""
|
|
1275
|
+
A simple decorator that demonstrates using CardDecoratorInjector
|
|
1276
|
+
to inject a card and render simple markdown content.
|
|
1277
|
+
"""
|
|
1278
|
+
...
|
|
1279
|
+
|
|
1280
|
+
@typing.overload
|
|
1281
|
+
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
1282
|
+
"""
|
|
1283
|
+
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
1284
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
1285
|
+
a Neo Cloud like Nebius.
|
|
1265
1286
|
"""
|
|
1266
1287
|
...
|
|
1267
1288
|
|
|
1268
1289
|
@typing.overload
|
|
1269
|
-
def
|
|
1270
|
-
...
|
|
1271
|
-
|
|
1272
|
-
@typing.overload
|
|
1273
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1290
|
+
def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
1274
1291
|
...
|
|
1275
1292
|
|
|
1276
|
-
def
|
|
1293
|
+
def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
1277
1294
|
"""
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
|
1283
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
|
1284
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
|
1285
|
-
|
|
1286
|
-
Note that all the values specified in parameters are added together so if you specify
|
|
1287
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
Parameters
|
|
1291
|
-
----------
|
|
1292
|
-
seconds : int, default 0
|
|
1293
|
-
Number of seconds to wait prior to timing out.
|
|
1294
|
-
minutes : int, default 0
|
|
1295
|
-
Number of minutes to wait prior to timing out.
|
|
1296
|
-
hours : int, default 0
|
|
1297
|
-
Number of hours to wait prior to timing out.
|
|
1295
|
+
Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
|
|
1296
|
+
It exists to make it easier for users to know that this decorator should only be used with
|
|
1297
|
+
a Neo Cloud like Nebius.
|
|
1298
1298
|
"""
|
|
1299
1299
|
...
|
|
1300
1300
|
|
|
@@ -1348,233 +1348,182 @@ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card
|
|
|
1348
1348
|
"""
|
|
1349
1349
|
...
|
|
1350
1350
|
|
|
1351
|
-
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1352
|
-
"""
|
|
1353
|
-
Specifies what flows belong to the same project.
|
|
1354
|
-
|
|
1355
|
-
A project-specific namespace is created for all flows that
|
|
1356
|
-
use the same `@project(name)`.
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
Parameters
|
|
1360
|
-
----------
|
|
1361
|
-
name : str
|
|
1362
|
-
Project name. Make sure that the name is unique amongst all
|
|
1363
|
-
projects that use the same production scheduler. The name may
|
|
1364
|
-
contain only lowercase alphanumeric characters and underscores.
|
|
1365
|
-
|
|
1366
|
-
branch : Optional[str], default None
|
|
1367
|
-
The branch to use. If not specified, the branch is set to
|
|
1368
|
-
`user.<username>` unless `production` is set to `True`. This can
|
|
1369
|
-
also be set on the command line using `--branch` as a top-level option.
|
|
1370
|
-
It is an error to specify `branch` in the decorator and on the command line.
|
|
1371
|
-
|
|
1372
|
-
production : bool, default False
|
|
1373
|
-
Whether or not the branch is the production branch. This can also be set on the
|
|
1374
|
-
command line using `--production` as a top-level option. It is an error to specify
|
|
1375
|
-
`production` in the decorator and on the command line.
|
|
1376
|
-
The project branch name will be:
|
|
1377
|
-
- if `branch` is specified:
|
|
1378
|
-
- if `production` is True: `prod.<branch>`
|
|
1379
|
-
- if `production` is False: `test.<branch>`
|
|
1380
|
-
- if `branch` is not specified:
|
|
1381
|
-
- if `production` is True: `prod`
|
|
1382
|
-
- if `production` is False: `user.<username>`
|
|
1383
|
-
"""
|
|
1384
|
-
...
|
|
1385
|
-
|
|
1386
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1387
|
-
"""
|
|
1388
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1389
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
Parameters
|
|
1393
|
-
----------
|
|
1394
|
-
timeout : int
|
|
1395
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1396
|
-
poke_interval : int
|
|
1397
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1398
|
-
mode : str
|
|
1399
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1400
|
-
exponential_backoff : bool
|
|
1401
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1402
|
-
pool : str
|
|
1403
|
-
the slot pool this task should run in,
|
|
1404
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1405
|
-
soft_fail : bool
|
|
1406
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1407
|
-
name : str
|
|
1408
|
-
Name of the sensor on Airflow
|
|
1409
|
-
description : str
|
|
1410
|
-
Description of sensor in the Airflow UI
|
|
1411
|
-
external_dag_id : str
|
|
1412
|
-
The dag_id that contains the task you want to wait for.
|
|
1413
|
-
external_task_ids : List[str]
|
|
1414
|
-
The list of task_ids that you want to wait for.
|
|
1415
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1416
|
-
allowed_states : List[str]
|
|
1417
|
-
Iterable of allowed states, (Default: ['success'])
|
|
1418
|
-
failed_states : List[str]
|
|
1419
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
|
1420
|
-
execution_delta : datetime.timedelta
|
|
1421
|
-
time difference with the previous execution to look at,
|
|
1422
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1423
|
-
check_existence: bool
|
|
1424
|
-
Set to True to check if the external task exists or check if
|
|
1425
|
-
the DAG to wait for exists. (Default: True)
|
|
1426
|
-
"""
|
|
1427
|
-
...
|
|
1428
|
-
|
|
1429
1351
|
@typing.overload
|
|
1430
|
-
def
|
|
1352
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1431
1353
|
"""
|
|
1432
|
-
Specifies the
|
|
1433
|
-
|
|
1354
|
+
Specifies the Conda environment for all steps of the flow.
|
|
1355
|
+
|
|
1356
|
+
Use `@conda_base` to set common libraries required by all
|
|
1357
|
+
steps and use `@conda` to specify step-specific additions.
|
|
1434
1358
|
|
|
1435
1359
|
|
|
1436
1360
|
Parameters
|
|
1437
1361
|
----------
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
1449
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
1362
|
+
packages : Dict[str, str], default {}
|
|
1363
|
+
Packages to use for this flow. The key is the name of the package
|
|
1364
|
+
and the value is the version to use.
|
|
1365
|
+
libraries : Dict[str, str], default {}
|
|
1366
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1367
|
+
python : str, optional, default None
|
|
1368
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1369
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1370
|
+
disabled : bool, default False
|
|
1371
|
+
If set to True, disables Conda.
|
|
1450
1372
|
"""
|
|
1451
1373
|
...
|
|
1452
1374
|
|
|
1453
1375
|
@typing.overload
|
|
1454
|
-
def
|
|
1376
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1455
1377
|
...
|
|
1456
1378
|
|
|
1457
|
-
def
|
|
1379
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
1458
1380
|
"""
|
|
1459
|
-
Specifies the
|
|
1460
|
-
|
|
1381
|
+
Specifies the Conda environment for all steps of the flow.
|
|
1382
|
+
|
|
1383
|
+
Use `@conda_base` to set common libraries required by all
|
|
1384
|
+
steps and use `@conda` to specify step-specific additions.
|
|
1461
1385
|
|
|
1462
1386
|
|
|
1463
1387
|
Parameters
|
|
1464
1388
|
----------
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
1476
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
1389
|
+
packages : Dict[str, str], default {}
|
|
1390
|
+
Packages to use for this flow. The key is the name of the package
|
|
1391
|
+
and the value is the version to use.
|
|
1392
|
+
libraries : Dict[str, str], default {}
|
|
1393
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1394
|
+
python : str, optional, default None
|
|
1395
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1396
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1397
|
+
disabled : bool, default False
|
|
1398
|
+
If set to True, disables Conda.
|
|
1477
1399
|
"""
|
|
1478
1400
|
...
|
|
1479
1401
|
|
|
1480
1402
|
@typing.overload
|
|
1481
|
-
def
|
|
1403
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1482
1404
|
"""
|
|
1483
|
-
Specifies the
|
|
1405
|
+
Specifies the event(s) that this flow depends on.
|
|
1484
1406
|
|
|
1485
1407
|
```
|
|
1486
|
-
@
|
|
1408
|
+
@trigger(event='foo')
|
|
1487
1409
|
```
|
|
1488
1410
|
or
|
|
1489
1411
|
```
|
|
1490
|
-
@
|
|
1412
|
+
@trigger(events=['foo', 'bar'])
|
|
1491
1413
|
```
|
|
1492
|
-
This decorator respects the @project decorator and triggers the flow
|
|
1493
|
-
when upstream runs within the same namespace complete successfully
|
|
1494
1414
|
|
|
1495
|
-
Additionally, you can specify
|
|
1496
|
-
|
|
1415
|
+
Additionally, you can specify the parameter mappings
|
|
1416
|
+
to map event payload to Metaflow parameters for the flow.
|
|
1497
1417
|
```
|
|
1498
|
-
@
|
|
1418
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1499
1419
|
```
|
|
1500
1420
|
or
|
|
1501
1421
|
```
|
|
1502
|
-
@
|
|
1422
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1423
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1503
1424
|
```
|
|
1504
1425
|
|
|
1505
|
-
|
|
1506
|
-
inferred from the current project or project branch):
|
|
1426
|
+
'parameters' can also be a list of strings and tuples like so:
|
|
1507
1427
|
```
|
|
1508
|
-
@
|
|
1428
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1429
|
+
```
|
|
1430
|
+
This is equivalent to:
|
|
1431
|
+
```
|
|
1432
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1509
1433
|
```
|
|
1510
|
-
|
|
1511
|
-
Note that `branch` is typically one of:
|
|
1512
|
-
- `prod`
|
|
1513
|
-
- `user.bob`
|
|
1514
|
-
- `test.my_experiment`
|
|
1515
|
-
- `prod.staging`
|
|
1516
1434
|
|
|
1517
1435
|
|
|
1518
1436
|
Parameters
|
|
1519
1437
|
----------
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1438
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
|
1439
|
+
Event dependency for this flow.
|
|
1440
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
|
1441
|
+
Events dependency for this flow.
|
|
1524
1442
|
options : Dict[str, Any], default {}
|
|
1525
1443
|
Backend-specific configuration for tuning eventing behavior.
|
|
1526
1444
|
"""
|
|
1527
1445
|
...
|
|
1528
1446
|
|
|
1529
1447
|
@typing.overload
|
|
1530
|
-
def
|
|
1448
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1531
1449
|
...
|
|
1532
1450
|
|
|
1533
|
-
def
|
|
1451
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
1534
1452
|
"""
|
|
1535
|
-
Specifies the
|
|
1453
|
+
Specifies the event(s) that this flow depends on.
|
|
1536
1454
|
|
|
1537
1455
|
```
|
|
1538
|
-
@
|
|
1456
|
+
@trigger(event='foo')
|
|
1539
1457
|
```
|
|
1540
1458
|
or
|
|
1541
1459
|
```
|
|
1542
|
-
@
|
|
1460
|
+
@trigger(events=['foo', 'bar'])
|
|
1543
1461
|
```
|
|
1544
|
-
This decorator respects the @project decorator and triggers the flow
|
|
1545
|
-
when upstream runs within the same namespace complete successfully
|
|
1546
1462
|
|
|
1547
|
-
Additionally, you can specify
|
|
1548
|
-
|
|
1463
|
+
Additionally, you can specify the parameter mappings
|
|
1464
|
+
to map event payload to Metaflow parameters for the flow.
|
|
1549
1465
|
```
|
|
1550
|
-
@
|
|
1466
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
|
1551
1467
|
```
|
|
1552
1468
|
or
|
|
1553
1469
|
```
|
|
1554
|
-
@
|
|
1470
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
|
1471
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1555
1472
|
```
|
|
1556
1473
|
|
|
1557
|
-
|
|
1558
|
-
inferred from the current project or project branch):
|
|
1474
|
+
'parameters' can also be a list of strings and tuples like so:
|
|
1559
1475
|
```
|
|
1560
|
-
@
|
|
1476
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1561
1477
|
```
|
|
1478
|
+
This is equivalent to:
|
|
1479
|
+
```
|
|
1480
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
|
1481
|
+
```
|
|
1482
|
+
|
|
1483
|
+
|
|
1484
|
+
Parameters
|
|
1485
|
+
----------
|
|
1486
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
|
1487
|
+
Event dependency for this flow.
|
|
1488
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
|
1489
|
+
Events dependency for this flow.
|
|
1490
|
+
options : Dict[str, Any], default {}
|
|
1491
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
1492
|
+
"""
|
|
1493
|
+
...
|
|
1494
|
+
|
|
1495
|
+
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1496
|
+
"""
|
|
1497
|
+
Specifies what flows belong to the same project.
|
|
1562
1498
|
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
- `user.bob`
|
|
1566
|
-
- `test.my_experiment`
|
|
1567
|
-
- `prod.staging`
|
|
1499
|
+
A project-specific namespace is created for all flows that
|
|
1500
|
+
use the same `@project(name)`.
|
|
1568
1501
|
|
|
1569
1502
|
|
|
1570
1503
|
Parameters
|
|
1571
1504
|
----------
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1505
|
+
name : str
|
|
1506
|
+
Project name. Make sure that the name is unique amongst all
|
|
1507
|
+
projects that use the same production scheduler. The name may
|
|
1508
|
+
contain only lowercase alphanumeric characters and underscores.
|
|
1509
|
+
|
|
1510
|
+
branch : Optional[str], default None
|
|
1511
|
+
The branch to use. If not specified, the branch is set to
|
|
1512
|
+
`user.<username>` unless `production` is set to `True`. This can
|
|
1513
|
+
also be set on the command line using `--branch` as a top-level option.
|
|
1514
|
+
It is an error to specify `branch` in the decorator and on the command line.
|
|
1515
|
+
|
|
1516
|
+
production : bool, default False
|
|
1517
|
+
Whether or not the branch is the production branch. This can also be set on the
|
|
1518
|
+
command line using `--production` as a top-level option. It is an error to specify
|
|
1519
|
+
`production` in the decorator and on the command line.
|
|
1520
|
+
The project branch name will be:
|
|
1521
|
+
- if `branch` is specified:
|
|
1522
|
+
- if `production` is True: `prod.<branch>`
|
|
1523
|
+
- if `production` is False: `test.<branch>`
|
|
1524
|
+
- if `branch` is not specified:
|
|
1525
|
+
- if `production` is True: `prod`
|
|
1526
|
+
- if `production` is False: `user.<username>`
|
|
1578
1527
|
"""
|
|
1579
1528
|
...
|
|
1580
1529
|
|
|
@@ -1692,6 +1641,49 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
|
|
|
1692
1641
|
"""
|
|
1693
1642
|
...
|
|
1694
1643
|
|
|
1644
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1645
|
+
"""
|
|
1646
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1647
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1648
|
+
|
|
1649
|
+
|
|
1650
|
+
Parameters
|
|
1651
|
+
----------
|
|
1652
|
+
timeout : int
|
|
1653
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1654
|
+
poke_interval : int
|
|
1655
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1656
|
+
mode : str
|
|
1657
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1658
|
+
exponential_backoff : bool
|
|
1659
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1660
|
+
pool : str
|
|
1661
|
+
the slot pool this task should run in,
|
|
1662
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1663
|
+
soft_fail : bool
|
|
1664
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1665
|
+
name : str
|
|
1666
|
+
Name of the sensor on Airflow
|
|
1667
|
+
description : str
|
|
1668
|
+
Description of sensor in the Airflow UI
|
|
1669
|
+
external_dag_id : str
|
|
1670
|
+
The dag_id that contains the task you want to wait for.
|
|
1671
|
+
external_task_ids : List[str]
|
|
1672
|
+
The list of task_ids that you want to wait for.
|
|
1673
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1674
|
+
allowed_states : List[str]
|
|
1675
|
+
Iterable of allowed states, (Default: ['success'])
|
|
1676
|
+
failed_states : List[str]
|
|
1677
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
|
1678
|
+
execution_delta : datetime.timedelta
|
|
1679
|
+
time difference with the previous execution to look at,
|
|
1680
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1681
|
+
check_existence: bool
|
|
1682
|
+
Set to True to check if the external task exists or check if
|
|
1683
|
+
the DAG to wait for exists. (Default: True)
|
|
1684
|
+
"""
|
|
1685
|
+
...
|
|
1686
|
+
|
|
1695
1687
|
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1696
1688
|
"""
|
|
1697
1689
|
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
|
@@ -1736,187 +1728,195 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
|
1736
1728
|
...
|
|
1737
1729
|
|
|
1738
1730
|
@typing.overload
|
|
1739
|
-
def
|
|
1731
|
+
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1740
1732
|
"""
|
|
1741
|
-
Specifies the
|
|
1733
|
+
Specifies the flow(s) that this flow depends on.
|
|
1742
1734
|
|
|
1743
1735
|
```
|
|
1744
|
-
@
|
|
1736
|
+
@trigger_on_finish(flow='FooFlow')
|
|
1745
1737
|
```
|
|
1746
1738
|
or
|
|
1747
1739
|
```
|
|
1748
|
-
@
|
|
1740
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
1749
1741
|
```
|
|
1742
|
+
This decorator respects the @project decorator and triggers the flow
|
|
1743
|
+
when upstream runs within the same namespace complete successfully
|
|
1750
1744
|
|
|
1751
|
-
Additionally, you can specify
|
|
1752
|
-
|
|
1745
|
+
Additionally, you can specify project aware upstream flow dependencies
|
|
1746
|
+
by specifying the fully qualified project_flow_name.
|
|
1753
1747
|
```
|
|
1754
|
-
@
|
|
1748
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
|
1755
1749
|
```
|
|
1756
1750
|
or
|
|
1757
1751
|
```
|
|
1758
|
-
@
|
|
1759
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1752
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
|
1760
1753
|
```
|
|
1761
1754
|
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1765
|
-
```
|
|
1766
|
-
This is equivalent to:
|
|
1755
|
+
You can also specify just the project or project branch (other values will be
|
|
1756
|
+
inferred from the current project or project branch):
|
|
1767
1757
|
```
|
|
1768
|
-
@
|
|
1758
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
|
1769
1759
|
```
|
|
1770
1760
|
|
|
1761
|
+
Note that `branch` is typically one of:
|
|
1762
|
+
- `prod`
|
|
1763
|
+
- `user.bob`
|
|
1764
|
+
- `test.my_experiment`
|
|
1765
|
+
- `prod.staging`
|
|
1766
|
+
|
|
1771
1767
|
|
|
1772
1768
|
Parameters
|
|
1773
1769
|
----------
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
1770
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
|
1771
|
+
Upstream flow dependency for this flow.
|
|
1772
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
|
1773
|
+
Upstream flow dependencies for this flow.
|
|
1778
1774
|
options : Dict[str, Any], default {}
|
|
1779
1775
|
Backend-specific configuration for tuning eventing behavior.
|
|
1780
1776
|
"""
|
|
1781
1777
|
...
|
|
1782
1778
|
|
|
1783
1779
|
@typing.overload
|
|
1784
|
-
def
|
|
1780
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1785
1781
|
...
|
|
1786
1782
|
|
|
1787
|
-
def
|
|
1783
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
1788
1784
|
"""
|
|
1789
|
-
Specifies the
|
|
1785
|
+
Specifies the flow(s) that this flow depends on.
|
|
1790
1786
|
|
|
1791
1787
|
```
|
|
1792
|
-
@
|
|
1788
|
+
@trigger_on_finish(flow='FooFlow')
|
|
1793
1789
|
```
|
|
1794
1790
|
or
|
|
1795
1791
|
```
|
|
1796
|
-
@
|
|
1792
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
1797
1793
|
```
|
|
1794
|
+
This decorator respects the @project decorator and triggers the flow
|
|
1795
|
+
when upstream runs within the same namespace complete successfully
|
|
1798
1796
|
|
|
1799
|
-
Additionally, you can specify
|
|
1800
|
-
|
|
1797
|
+
Additionally, you can specify project aware upstream flow dependencies
|
|
1798
|
+
by specifying the fully qualified project_flow_name.
|
|
1801
1799
|
```
|
|
1802
|
-
@
|
|
1800
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
|
1803
1801
|
```
|
|
1804
1802
|
or
|
|
1805
1803
|
```
|
|
1806
|
-
@
|
|
1807
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
|
1804
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
|
1808
1805
|
```
|
|
1809
1806
|
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
|
1813
|
-
```
|
|
1814
|
-
This is equivalent to:
|
|
1807
|
+
You can also specify just the project or project branch (other values will be
|
|
1808
|
+
inferred from the current project or project branch):
|
|
1815
1809
|
```
|
|
1816
|
-
@
|
|
1810
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
|
1817
1811
|
```
|
|
1818
1812
|
|
|
1813
|
+
Note that `branch` is typically one of:
|
|
1814
|
+
- `prod`
|
|
1815
|
+
- `user.bob`
|
|
1816
|
+
- `test.my_experiment`
|
|
1817
|
+
- `prod.staging`
|
|
1818
|
+
|
|
1819
1819
|
|
|
1820
1820
|
Parameters
|
|
1821
1821
|
----------
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1822
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
|
1823
|
+
Upstream flow dependency for this flow.
|
|
1824
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
|
1825
|
+
Upstream flow dependencies for this flow.
|
|
1826
1826
|
options : Dict[str, Any], default {}
|
|
1827
1827
|
Backend-specific configuration for tuning eventing behavior.
|
|
1828
1828
|
"""
|
|
1829
1829
|
...
|
|
1830
1830
|
|
|
1831
1831
|
@typing.overload
|
|
1832
|
-
def
|
|
1832
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1833
1833
|
"""
|
|
1834
|
-
Specifies the
|
|
1835
|
-
|
|
1836
|
-
Use `@conda_base` to set common libraries required by all
|
|
1837
|
-
steps and use `@conda` to specify step-specific additions.
|
|
1834
|
+
Specifies the PyPI packages for all steps of the flow.
|
|
1838
1835
|
|
|
1836
|
+
Use `@pypi_base` to set common packages required by all
|
|
1837
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1839
1838
|
|
|
1840
1839
|
Parameters
|
|
1841
1840
|
----------
|
|
1842
|
-
packages : Dict[str, str], default {}
|
|
1841
|
+
packages : Dict[str, str], default: {}
|
|
1843
1842
|
Packages to use for this flow. The key is the name of the package
|
|
1844
1843
|
and the value is the version to use.
|
|
1845
|
-
|
|
1846
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1847
|
-
python : str, optional, default None
|
|
1844
|
+
python : str, optional, default: None
|
|
1848
1845
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1849
1846
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1850
|
-
disabled : bool, default False
|
|
1851
|
-
If set to True, disables Conda.
|
|
1852
1847
|
"""
|
|
1853
1848
|
...
|
|
1854
1849
|
|
|
1855
1850
|
@typing.overload
|
|
1856
|
-
def
|
|
1851
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1857
1852
|
...
|
|
1858
1853
|
|
|
1859
|
-
def
|
|
1854
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
1860
1855
|
"""
|
|
1861
|
-
Specifies the
|
|
1862
|
-
|
|
1863
|
-
Use `@conda_base` to set common libraries required by all
|
|
1864
|
-
steps and use `@conda` to specify step-specific additions.
|
|
1856
|
+
Specifies the PyPI packages for all steps of the flow.
|
|
1865
1857
|
|
|
1858
|
+
Use `@pypi_base` to set common packages required by all
|
|
1859
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
1866
1860
|
|
|
1867
1861
|
Parameters
|
|
1868
1862
|
----------
|
|
1869
|
-
packages : Dict[str, str], default {}
|
|
1863
|
+
packages : Dict[str, str], default: {}
|
|
1870
1864
|
Packages to use for this flow. The key is the name of the package
|
|
1871
1865
|
and the value is the version to use.
|
|
1872
|
-
|
|
1873
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1874
|
-
python : str, optional, default None
|
|
1866
|
+
python : str, optional, default: None
|
|
1875
1867
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1876
1868
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1877
|
-
disabled : bool, default False
|
|
1878
|
-
If set to True, disables Conda.
|
|
1879
1869
|
"""
|
|
1880
1870
|
...
|
|
1881
1871
|
|
|
1882
1872
|
@typing.overload
|
|
1883
|
-
def
|
|
1873
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1884
1874
|
"""
|
|
1885
|
-
Specifies the
|
|
1875
|
+
Specifies the times when the flow should be run when running on a
|
|
1876
|
+
production scheduler.
|
|
1886
1877
|
|
|
1887
|
-
Use `@pypi_base` to set common packages required by all
|
|
1888
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
1889
1878
|
|
|
1890
1879
|
Parameters
|
|
1891
1880
|
----------
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1881
|
+
hourly : bool, default False
|
|
1882
|
+
Run the workflow hourly.
|
|
1883
|
+
daily : bool, default True
|
|
1884
|
+
Run the workflow daily.
|
|
1885
|
+
weekly : bool, default False
|
|
1886
|
+
Run the workflow weekly.
|
|
1887
|
+
cron : str, optional, default None
|
|
1888
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
|
1889
|
+
specified by this expression.
|
|
1890
|
+
timezone : str, optional, default None
|
|
1891
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
1892
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
1898
1893
|
"""
|
|
1899
1894
|
...
|
|
1900
1895
|
|
|
1901
1896
|
@typing.overload
|
|
1902
|
-
def
|
|
1897
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1903
1898
|
...
|
|
1904
1899
|
|
|
1905
|
-
def
|
|
1900
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
|
1906
1901
|
"""
|
|
1907
|
-
Specifies the
|
|
1902
|
+
Specifies the times when the flow should be run when running on a
|
|
1903
|
+
production scheduler.
|
|
1908
1904
|
|
|
1909
|
-
Use `@pypi_base` to set common packages required by all
|
|
1910
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
1911
1905
|
|
|
1912
1906
|
Parameters
|
|
1913
1907
|
----------
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
1919
|
-
|
|
1908
|
+
hourly : bool, default False
|
|
1909
|
+
Run the workflow hourly.
|
|
1910
|
+
daily : bool, default True
|
|
1911
|
+
Run the workflow daily.
|
|
1912
|
+
weekly : bool, default False
|
|
1913
|
+
Run the workflow weekly.
|
|
1914
|
+
cron : str, optional, default None
|
|
1915
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
|
1916
|
+
specified by this expression.
|
|
1917
|
+
timezone : str, optional, default None
|
|
1918
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
|
1919
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
|
1920
1920
|
"""
|
|
1921
1921
|
...
|
|
1922
1922
|
|