ob-metaflow-stubs 6.0.3.170__py2.py3-none-any.whl → 6.0.3.171__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +596 -592
- metaflow-stubs/cards.pyi +1 -1
- metaflow-stubs/cli.pyi +1 -1
- metaflow-stubs/cli_components/__init__.pyi +1 -1
- metaflow-stubs/cli_components/utils.pyi +1 -1
- metaflow-stubs/client/__init__.pyi +1 -1
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +1 -1
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +1 -1
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +3 -3
- metaflow-stubs/info_file.pyi +1 -1
- metaflow-stubs/metadata_provider/__init__.pyi +1 -1
- metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +1 -1
- metaflow-stubs/metaflow_config.pyi +1 -1
- metaflow-stubs/metaflow_current.pyi +48 -48
- metaflow-stubs/metaflow_git.pyi +1 -1
- metaflow-stubs/mf_extensions/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +12 -0
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +30 -0
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +15 -7
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
- metaflow-stubs/multicore_utils.pyi +1 -1
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +9 -9
- metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
- metaflow-stubs/plugins/airflow/exception.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
- metaflow-stubs/plugins/argo/__init__.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
- metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/aws/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
- metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
- metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/cards/__init__.pyi +1 -1
- metaflow-stubs/plugins/cards/card_client.pyi +1 -1
- metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
- metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
- metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
- metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
- metaflow-stubs/plugins/cards/exception.pyi +1 -1
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/local.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
- metaflow-stubs/plugins/debug_logger.pyi +1 -1
- metaflow-stubs/plugins/debug_monitor.pyi +1 -1
- metaflow-stubs/plugins/environment_decorator.pyi +1 -1
- metaflow-stubs/plugins/events_decorator.pyi +1 -1
- metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
- metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
- metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
- metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
- metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
- metaflow-stubs/plugins/ollama/__init__.pyi +17 -9
- metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
- metaflow-stubs/plugins/perimeters.pyi +1 -1
- metaflow-stubs/plugins/project_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
- metaflow-stubs/plugins/pypi/utils.pyi +1 -1
- metaflow-stubs/plugins/resources_decorator.pyi +1 -1
- metaflow-stubs/plugins/retry_decorator.pyi +1 -1
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
- metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
- metaflow-stubs/plugins/storage_executor.pyi +1 -1
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/plugins/uv/__init__.pyi +1 -1
- metaflow-stubs/plugins/uv/uv_environment.pyi +1 -1
- metaflow-stubs/profilers/__init__.pyi +1 -1
- metaflow-stubs/pylint_wrapper.pyi +1 -1
- metaflow-stubs/runner/__init__.pyi +1 -1
- metaflow-stubs/runner/deployer.pyi +6 -6
- metaflow-stubs/runner/deployer_impl.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +2 -2
- metaflow-stubs/runner/nbdeploy.pyi +1 -1
- metaflow-stubs/runner/nbrun.pyi +1 -1
- metaflow-stubs/runner/subprocess_manager.pyi +1 -1
- metaflow-stubs/runner/utils.pyi +1 -1
- metaflow-stubs/system/__init__.pyi +1 -1
- metaflow-stubs/system/system_logger.pyi +1 -1
- metaflow-stubs/system/system_monitor.pyi +1 -1
- metaflow-stubs/tagging_util.pyi +1 -1
- metaflow-stubs/tuple_util.pyi +1 -1
- metaflow-stubs/user_configs/__init__.pyi +1 -1
- metaflow-stubs/user_configs/config_decorators.pyi +5 -5
- metaflow-stubs/user_configs/config_options.pyi +3 -3
- metaflow-stubs/user_configs/config_parameters.pyi +5 -5
- {ob_metaflow_stubs-6.0.3.170.dist-info → ob_metaflow_stubs-6.0.3.171.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.171.dist-info/RECORD +208 -0
- ob_metaflow_stubs-6.0.3.170.dist-info/RECORD +0 -206
- {ob_metaflow_stubs-6.0.3.170.dist-info → ob_metaflow_stubs-6.0.3.171.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.170.dist-info → ob_metaflow_stubs-6.0.3.171.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
3
|
# MF version: 2.15.14.1+obcheckpoint(0.2.1);ob(v1) #
|
4
|
-
# Generated on 2025-05-
|
4
|
+
# Generated on 2025-05-27T00:40:42.234892 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -45,8 +45,8 @@ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package imp
|
|
45
45
|
from . import includefile as includefile
|
46
46
|
from .includefile import IncludeFile as IncludeFile
|
47
47
|
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
48
|
-
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
49
48
|
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
49
|
+
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
50
50
|
from . import client as client
|
51
51
|
from .client.core import namespace as namespace
|
52
52
|
from .client.core import get_namespace as get_namespace
|
@@ -153,218 +153,172 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
153
153
|
"""
|
154
154
|
...
|
155
155
|
|
156
|
-
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
157
|
-
"""
|
158
|
-
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
159
|
-
|
160
|
-
|
161
|
-
Parameters
|
162
|
-
----------
|
163
|
-
temp_dir_root : str, optional
|
164
|
-
The root directory that will hold the temporary directory where objects will be downloaded.
|
165
|
-
|
166
|
-
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
167
|
-
The list of repos (models/datasets) to load.
|
168
|
-
|
169
|
-
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
170
|
-
|
171
|
-
- If repo (model/dataset) is not found in the datastore:
|
172
|
-
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
173
|
-
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
174
|
-
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
175
|
-
|
176
|
-
- If repo is found in the datastore:
|
177
|
-
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
178
|
-
"""
|
179
|
-
...
|
180
|
-
|
181
|
-
@typing.overload
|
182
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
183
|
-
"""
|
184
|
-
Specifies environment variables to be set prior to the execution of a step.
|
185
|
-
|
186
|
-
|
187
|
-
Parameters
|
188
|
-
----------
|
189
|
-
vars : Dict[str, str], default {}
|
190
|
-
Dictionary of environment variables to set.
|
191
|
-
"""
|
192
|
-
...
|
193
|
-
|
194
156
|
@typing.overload
|
195
|
-
def
|
196
|
-
...
|
197
|
-
|
198
|
-
@typing.overload
|
199
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
200
|
-
...
|
201
|
-
|
202
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
157
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
203
158
|
"""
|
204
|
-
Specifies
|
205
|
-
|
159
|
+
Specifies the number of times the task corresponding
|
160
|
+
to a step needs to be retried.
|
206
161
|
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
Dictionary of environment variables to set.
|
211
|
-
"""
|
212
|
-
...
|
213
|
-
|
214
|
-
@typing.overload
|
215
|
-
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
216
|
-
"""
|
217
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
162
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
163
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
164
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
218
165
|
|
219
|
-
|
166
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
167
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
168
|
+
ensuring that the flow execution can continue.
|
220
169
|
|
221
170
|
|
222
171
|
Parameters
|
223
172
|
----------
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
options : Dict[str, Any], default {}
|
229
|
-
Options passed to the card. The contents depend on the card type.
|
230
|
-
timeout : int, default 45
|
231
|
-
Interrupt reporting if it takes more than this many seconds.
|
173
|
+
times : int, default 3
|
174
|
+
Number of times to retry this task.
|
175
|
+
minutes_between_retries : int, default 2
|
176
|
+
Number of minutes between retries.
|
232
177
|
"""
|
233
178
|
...
|
234
179
|
|
235
180
|
@typing.overload
|
236
|
-
def
|
181
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
237
182
|
...
|
238
183
|
|
239
184
|
@typing.overload
|
240
|
-
def
|
185
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
241
186
|
...
|
242
187
|
|
243
|
-
def
|
188
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
244
189
|
"""
|
245
|
-
|
246
|
-
|
247
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
190
|
+
Specifies the number of times the task corresponding
|
191
|
+
to a step needs to be retried.
|
248
192
|
|
193
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
194
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
195
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
249
196
|
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
Card type.
|
254
|
-
id : str, optional, default None
|
255
|
-
If multiple cards are present, use this id to identify this card.
|
256
|
-
options : Dict[str, Any], default {}
|
257
|
-
Options passed to the card. The contents depend on the card type.
|
258
|
-
timeout : int, default 45
|
259
|
-
Interrupt reporting if it takes more than this many seconds.
|
260
|
-
"""
|
261
|
-
...
|
262
|
-
|
263
|
-
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
264
|
-
"""
|
265
|
-
Specifies that this step should execute on DGX cloud.
|
197
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
198
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
199
|
+
ensuring that the flow execution can continue.
|
266
200
|
|
267
201
|
|
268
202
|
Parameters
|
269
203
|
----------
|
270
|
-
|
271
|
-
Number of
|
272
|
-
|
273
|
-
|
274
|
-
queue_timeout : int
|
275
|
-
Time to keep the job in NVCF's queue.
|
204
|
+
times : int, default 3
|
205
|
+
Number of times to retry this task.
|
206
|
+
minutes_between_retries : int, default 2
|
207
|
+
Number of minutes between retries.
|
276
208
|
"""
|
277
209
|
...
|
278
210
|
|
279
211
|
@typing.overload
|
280
|
-
def
|
212
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
281
213
|
"""
|
282
|
-
|
283
|
-
|
214
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
215
|
+
the execution of a step.
|
284
216
|
|
285
217
|
|
286
218
|
Parameters
|
287
219
|
----------
|
288
|
-
|
289
|
-
|
290
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
291
|
-
- `current.checkpoint`
|
292
|
-
- `current.model`
|
293
|
-
- `current.huggingface_hub`
|
294
|
-
|
295
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
296
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
297
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
298
|
-
|
299
|
-
temp_dir_root : str, default: None
|
300
|
-
The root directory under which `current.model.loaded` will store loaded models
|
220
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
221
|
+
List of secret specs, defining how the secrets are to be retrieved
|
301
222
|
"""
|
302
223
|
...
|
303
224
|
|
304
225
|
@typing.overload
|
305
|
-
def
|
226
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
306
227
|
...
|
307
228
|
|
308
229
|
@typing.overload
|
309
|
-
def
|
230
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
310
231
|
...
|
311
232
|
|
312
|
-
def
|
233
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
313
234
|
"""
|
314
|
-
|
315
|
-
|
235
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
236
|
+
the execution of a step.
|
316
237
|
|
317
238
|
|
318
239
|
Parameters
|
319
240
|
----------
|
320
|
-
|
321
|
-
|
322
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
323
|
-
- `current.checkpoint`
|
324
|
-
- `current.model`
|
325
|
-
- `current.huggingface_hub`
|
326
|
-
|
327
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
328
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
329
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
330
|
-
|
331
|
-
temp_dir_root : str, default: None
|
332
|
-
The root directory under which `current.model.loaded` will store loaded models
|
241
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
242
|
+
List of secret specs, defining how the secrets are to be retrieved
|
333
243
|
"""
|
334
244
|
...
|
335
245
|
|
336
246
|
@typing.overload
|
337
|
-
def
|
247
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
338
248
|
"""
|
339
|
-
Specifies
|
340
|
-
|
249
|
+
Specifies the resources needed when executing this step.
|
250
|
+
|
251
|
+
Use `@resources` to specify the resource requirements
|
252
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
253
|
+
|
254
|
+
You can choose the compute layer on the command line by executing e.g.
|
255
|
+
```
|
256
|
+
python myflow.py run --with batch
|
257
|
+
```
|
258
|
+
or
|
259
|
+
```
|
260
|
+
python myflow.py run --with kubernetes
|
261
|
+
```
|
262
|
+
which executes the flow on the desired system using the
|
263
|
+
requirements specified in `@resources`.
|
341
264
|
|
342
265
|
|
343
266
|
Parameters
|
344
267
|
----------
|
345
|
-
|
346
|
-
|
268
|
+
cpu : int, default 1
|
269
|
+
Number of CPUs required for this step.
|
270
|
+
gpu : int, optional, default None
|
271
|
+
Number of GPUs required for this step.
|
272
|
+
disk : int, optional, default None
|
273
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
274
|
+
memory : int, default 4096
|
275
|
+
Memory size (in MB) required for this step.
|
276
|
+
shared_memory : int, optional, default None
|
277
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
278
|
+
This parameter maps to the `--shm-size` option in Docker.
|
347
279
|
"""
|
348
280
|
...
|
349
281
|
|
350
282
|
@typing.overload
|
351
|
-
def
|
283
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
352
284
|
...
|
353
285
|
|
354
286
|
@typing.overload
|
355
|
-
def
|
287
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
356
288
|
...
|
357
289
|
|
358
|
-
def
|
290
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
359
291
|
"""
|
360
|
-
Specifies
|
361
|
-
|
292
|
+
Specifies the resources needed when executing this step.
|
293
|
+
|
294
|
+
Use `@resources` to specify the resource requirements
|
295
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
296
|
+
|
297
|
+
You can choose the compute layer on the command line by executing e.g.
|
298
|
+
```
|
299
|
+
python myflow.py run --with batch
|
300
|
+
```
|
301
|
+
or
|
302
|
+
```
|
303
|
+
python myflow.py run --with kubernetes
|
304
|
+
```
|
305
|
+
which executes the flow on the desired system using the
|
306
|
+
requirements specified in `@resources`.
|
362
307
|
|
363
308
|
|
364
309
|
Parameters
|
365
310
|
----------
|
366
|
-
|
367
|
-
|
311
|
+
cpu : int, default 1
|
312
|
+
Number of CPUs required for this step.
|
313
|
+
gpu : int, optional, default None
|
314
|
+
Number of GPUs required for this step.
|
315
|
+
disk : int, optional, default None
|
316
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
317
|
+
memory : int, default 4096
|
318
|
+
Memory size (in MB) required for this step.
|
319
|
+
shared_memory : int, optional, default None
|
320
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
321
|
+
This parameter maps to the `--shm-size` option in Docker.
|
368
322
|
"""
|
369
323
|
...
|
370
324
|
|
@@ -427,79 +381,164 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
427
381
|
"""
|
428
382
|
...
|
429
383
|
|
430
|
-
|
431
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
432
|
-
"""
|
433
|
-
Internal decorator to support Fast bakery
|
434
|
-
"""
|
435
|
-
...
|
436
|
-
|
437
|
-
@typing.overload
|
438
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
439
|
-
...
|
440
|
-
|
441
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
442
|
-
"""
|
443
|
-
Internal decorator to support Fast bakery
|
384
|
+
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
444
385
|
"""
|
445
|
-
|
386
|
+
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
387
|
+
|
388
|
+
|
389
|
+
Parameters
|
390
|
+
----------
|
391
|
+
temp_dir_root : str, optional
|
392
|
+
The root directory that will hold the temporary directory where objects will be downloaded.
|
393
|
+
|
394
|
+
load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
|
395
|
+
The list of repos (models/datasets) to load.
|
396
|
+
|
397
|
+
Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
|
398
|
+
|
399
|
+
- If repo (model/dataset) is not found in the datastore:
|
400
|
+
- Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
|
401
|
+
- Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
|
402
|
+
- All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
|
403
|
+
|
404
|
+
- If repo is found in the datastore:
|
405
|
+
- Loads it directly from datastore to local path (can be temporary directory or specified path)
|
406
|
+
"""
|
407
|
+
...
|
408
|
+
|
409
|
+
def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
410
|
+
"""
|
411
|
+
Specifies that this step is used to deploy an instance of the app.
|
412
|
+
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
413
|
+
|
414
|
+
|
415
|
+
Parameters
|
416
|
+
----------
|
417
|
+
app_port : int
|
418
|
+
Number of GPUs to use.
|
419
|
+
app_name : str
|
420
|
+
Name of the app to deploy.
|
421
|
+
"""
|
422
|
+
...
|
423
|
+
|
424
|
+
def ollama(*, models: list, backend: str, force_pull: bool, skip_push_check: bool, debug: bool) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
425
|
+
"""
|
426
|
+
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
427
|
+
|
428
|
+
User code call
|
429
|
+
--------------
|
430
|
+
@ollama(
|
431
|
+
models=[...],
|
432
|
+
...
|
433
|
+
)
|
434
|
+
|
435
|
+
Valid backend options
|
436
|
+
---------------------
|
437
|
+
- 'local': Run as a separate process on the local task machine.
|
438
|
+
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
439
|
+
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
440
|
+
|
441
|
+
Valid model options
|
442
|
+
-------------------
|
443
|
+
Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
|
444
|
+
|
445
|
+
|
446
|
+
Parameters
|
447
|
+
----------
|
448
|
+
models: list[str]
|
449
|
+
List of Ollama containers running models in sidecars.
|
450
|
+
backend: str
|
451
|
+
Determines where and how to run the Ollama process.
|
452
|
+
force_pull: bool
|
453
|
+
Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
|
454
|
+
skip_push_check: bool
|
455
|
+
Whether to skip the check that populates/overwrites remote cache on terminating an ollama model.
|
456
|
+
debug: bool
|
457
|
+
Whether to turn on verbose debugging logs.
|
458
|
+
"""
|
459
|
+
...
|
446
460
|
|
447
461
|
@typing.overload
|
448
|
-
def
|
462
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
449
463
|
"""
|
450
|
-
Specifies
|
464
|
+
Specifies environment variables to be set prior to the execution of a step.
|
465
|
+
|
466
|
+
|
467
|
+
Parameters
|
468
|
+
----------
|
469
|
+
vars : Dict[str, str], default {}
|
470
|
+
Dictionary of environment variables to set.
|
471
|
+
"""
|
472
|
+
...
|
473
|
+
|
474
|
+
@typing.overload
|
475
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
476
|
+
...
|
477
|
+
|
478
|
+
@typing.overload
|
479
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
480
|
+
...
|
481
|
+
|
482
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
483
|
+
"""
|
484
|
+
Specifies environment variables to be set prior to the execution of a step.
|
485
|
+
|
486
|
+
|
487
|
+
Parameters
|
488
|
+
----------
|
489
|
+
vars : Dict[str, str], default {}
|
490
|
+
Dictionary of environment variables to set.
|
491
|
+
"""
|
492
|
+
...
|
493
|
+
|
494
|
+
@typing.overload
|
495
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
496
|
+
"""
|
497
|
+
Specifies the PyPI packages for the step.
|
451
498
|
|
452
499
|
Information in this decorator will augment any
|
453
|
-
attributes set in the `@
|
454
|
-
you can use `@
|
455
|
-
steps and use `@
|
500
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
501
|
+
you can use `@pypi_base` to set packages required by all
|
502
|
+
steps and use `@pypi` to specify step-specific overrides.
|
456
503
|
|
457
504
|
|
458
505
|
Parameters
|
459
506
|
----------
|
460
|
-
packages : Dict[str, str], default {}
|
507
|
+
packages : Dict[str, str], default: {}
|
461
508
|
Packages to use for this step. The key is the name of the package
|
462
509
|
and the value is the version to use.
|
463
|
-
|
464
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
465
|
-
python : str, optional, default None
|
510
|
+
python : str, optional, default: None
|
466
511
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
467
512
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
468
|
-
disabled : bool, default False
|
469
|
-
If set to True, disables @conda.
|
470
513
|
"""
|
471
514
|
...
|
472
515
|
|
473
516
|
@typing.overload
|
474
|
-
def
|
517
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
475
518
|
...
|
476
519
|
|
477
520
|
@typing.overload
|
478
|
-
def
|
521
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
479
522
|
...
|
480
523
|
|
481
|
-
def
|
524
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
482
525
|
"""
|
483
|
-
Specifies the
|
526
|
+
Specifies the PyPI packages for the step.
|
484
527
|
|
485
528
|
Information in this decorator will augment any
|
486
|
-
attributes set in the `@
|
487
|
-
you can use `@
|
488
|
-
steps and use `@
|
529
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
530
|
+
you can use `@pypi_base` to set packages required by all
|
531
|
+
steps and use `@pypi` to specify step-specific overrides.
|
489
532
|
|
490
533
|
|
491
534
|
Parameters
|
492
535
|
----------
|
493
|
-
packages : Dict[str, str], default {}
|
536
|
+
packages : Dict[str, str], default: {}
|
494
537
|
Packages to use for this step. The key is the name of the package
|
495
538
|
and the value is the version to use.
|
496
|
-
|
497
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
498
|
-
python : str, optional, default None
|
539
|
+
python : str, optional, default: None
|
499
540
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
500
541
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
501
|
-
disabled : bool, default False
|
502
|
-
If set to True, disables @conda.
|
503
542
|
"""
|
504
543
|
...
|
505
544
|
|
@@ -555,325 +594,200 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
555
594
|
...
|
556
595
|
|
557
596
|
@typing.overload
|
558
|
-
def
|
597
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
559
598
|
"""
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
|
565
|
-
|
599
|
+
Decorator prototype for all step decorators. This function gets specialized
|
600
|
+
and imported for all decorators types by _import_plugin_decorators().
|
601
|
+
"""
|
602
|
+
...
|
603
|
+
|
604
|
+
@typing.overload
|
605
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
606
|
+
...
|
607
|
+
|
608
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
609
|
+
"""
|
610
|
+
Decorator prototype for all step decorators. This function gets specialized
|
611
|
+
and imported for all decorators types by _import_plugin_decorators().
|
612
|
+
"""
|
613
|
+
...
|
614
|
+
|
615
|
+
@typing.overload
|
616
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
617
|
+
"""
|
618
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
566
619
|
|
567
|
-
|
568
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
569
|
-
ensuring that the flow execution can continue.
|
620
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
570
621
|
|
571
622
|
|
572
623
|
Parameters
|
573
624
|
----------
|
574
|
-
|
575
|
-
|
576
|
-
|
577
|
-
|
625
|
+
type : str, default 'default'
|
626
|
+
Card type.
|
627
|
+
id : str, optional, default None
|
628
|
+
If multiple cards are present, use this id to identify this card.
|
629
|
+
options : Dict[str, Any], default {}
|
630
|
+
Options passed to the card. The contents depend on the card type.
|
631
|
+
timeout : int, default 45
|
632
|
+
Interrupt reporting if it takes more than this many seconds.
|
578
633
|
"""
|
579
634
|
...
|
580
635
|
|
581
636
|
@typing.overload
|
582
|
-
def
|
637
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
583
638
|
...
|
584
639
|
|
585
640
|
@typing.overload
|
586
|
-
def
|
641
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
587
642
|
...
|
588
643
|
|
589
|
-
def
|
644
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
590
645
|
"""
|
591
|
-
|
592
|
-
to a step needs to be retried.
|
593
|
-
|
594
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
595
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
596
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
646
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
597
647
|
|
598
|
-
|
599
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
600
|
-
ensuring that the flow execution can continue.
|
648
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
601
649
|
|
602
650
|
|
603
651
|
Parameters
|
604
652
|
----------
|
605
|
-
|
606
|
-
|
607
|
-
|
608
|
-
|
653
|
+
type : str, default 'default'
|
654
|
+
Card type.
|
655
|
+
id : str, optional, default None
|
656
|
+
If multiple cards are present, use this id to identify this card.
|
657
|
+
options : Dict[str, Any], default {}
|
658
|
+
Options passed to the card. The contents depend on the card type.
|
659
|
+
timeout : int, default 45
|
660
|
+
Interrupt reporting if it takes more than this many seconds.
|
609
661
|
"""
|
610
662
|
...
|
611
663
|
|
612
664
|
@typing.overload
|
613
|
-
def
|
665
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
614
666
|
"""
|
615
|
-
|
667
|
+
Enables loading / saving of models within a step.
|
616
668
|
|
617
|
-
Information in this decorator will augment any
|
618
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
619
|
-
you can use `@pypi_base` to set packages required by all
|
620
|
-
steps and use `@pypi` to specify step-specific overrides.
|
621
669
|
|
622
670
|
|
623
671
|
Parameters
|
624
672
|
----------
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
673
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
674
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
675
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
676
|
+
- `current.checkpoint`
|
677
|
+
- `current.model`
|
678
|
+
- `current.huggingface_hub`
|
679
|
+
|
680
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
681
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
682
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
683
|
+
|
684
|
+
temp_dir_root : str, default: None
|
685
|
+
The root directory under which `current.model.loaded` will store loaded models
|
631
686
|
"""
|
632
687
|
...
|
633
688
|
|
634
689
|
@typing.overload
|
635
|
-
def
|
690
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
636
691
|
...
|
637
692
|
|
638
693
|
@typing.overload
|
639
|
-
def
|
694
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
640
695
|
...
|
641
696
|
|
642
|
-
def
|
697
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
643
698
|
"""
|
644
|
-
|
699
|
+
Enables loading / saving of models within a step.
|
645
700
|
|
646
|
-
Information in this decorator will augment any
|
647
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
648
|
-
you can use `@pypi_base` to set packages required by all
|
649
|
-
steps and use `@pypi` to specify step-specific overrides.
|
650
701
|
|
651
702
|
|
652
703
|
Parameters
|
653
704
|
----------
|
654
|
-
|
655
|
-
|
656
|
-
|
657
|
-
|
658
|
-
|
659
|
-
|
660
|
-
"""
|
661
|
-
...
|
662
|
-
|
663
|
-
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
664
|
-
"""
|
665
|
-
Specifies that this step should execute on DGX cloud.
|
666
|
-
|
667
|
-
|
668
|
-
Parameters
|
669
|
-
----------
|
670
|
-
gpu : int
|
671
|
-
Number of GPUs to use.
|
672
|
-
gpu_type : str
|
673
|
-
Type of Nvidia GPU to use.
|
674
|
-
"""
|
675
|
-
...
|
676
|
-
|
677
|
-
def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
678
|
-
"""
|
679
|
-
This decorator is used to run Ollama APIs as Metaflow task sidecars.
|
680
|
-
|
681
|
-
User code call
|
682
|
-
-----------
|
683
|
-
@ollama(
|
684
|
-
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
685
|
-
backend='local'
|
686
|
-
)
|
687
|
-
|
688
|
-
Valid backend options
|
689
|
-
---------------------
|
690
|
-
- 'local': Run as a separate process on the local task machine.
|
691
|
-
- (TODO) 'managed': Outerbounds hosts and selects compute provider.
|
692
|
-
- (TODO) 'remote': Spin up separate instance to serve Ollama models.
|
693
|
-
|
694
|
-
Valid model options
|
695
|
-
----------------
|
696
|
-
- 'llama3.2'
|
697
|
-
- 'llama3.3'
|
698
|
-
- any model here https://ollama.com/search
|
699
|
-
|
700
|
-
|
701
|
-
Parameters
|
702
|
-
----------
|
703
|
-
models: list[Ollama]
|
704
|
-
List of Ollama containers running models in sidecars.
|
705
|
-
backend: str
|
706
|
-
Determines where and how to run the Ollama process.
|
707
|
-
"""
|
708
|
-
...
|
709
|
-
|
710
|
-
@typing.overload
|
711
|
-
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
712
|
-
"""
|
713
|
-
Specifies the resources needed when executing this step.
|
714
|
-
|
715
|
-
Use `@resources` to specify the resource requirements
|
716
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
717
|
-
|
718
|
-
You can choose the compute layer on the command line by executing e.g.
|
719
|
-
```
|
720
|
-
python myflow.py run --with batch
|
721
|
-
```
|
722
|
-
or
|
723
|
-
```
|
724
|
-
python myflow.py run --with kubernetes
|
725
|
-
```
|
726
|
-
which executes the flow on the desired system using the
|
727
|
-
requirements specified in `@resources`.
|
728
|
-
|
729
|
-
|
730
|
-
Parameters
|
731
|
-
----------
|
732
|
-
cpu : int, default 1
|
733
|
-
Number of CPUs required for this step.
|
734
|
-
gpu : int, optional, default None
|
735
|
-
Number of GPUs required for this step.
|
736
|
-
disk : int, optional, default None
|
737
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
738
|
-
memory : int, default 4096
|
739
|
-
Memory size (in MB) required for this step.
|
740
|
-
shared_memory : int, optional, default None
|
741
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
742
|
-
This parameter maps to the `--shm-size` option in Docker.
|
743
|
-
"""
|
744
|
-
...
|
745
|
-
|
746
|
-
@typing.overload
|
747
|
-
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
748
|
-
...
|
749
|
-
|
750
|
-
@typing.overload
|
751
|
-
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
752
|
-
...
|
753
|
-
|
754
|
-
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
755
|
-
"""
|
756
|
-
Specifies the resources needed when executing this step.
|
757
|
-
|
758
|
-
Use `@resources` to specify the resource requirements
|
759
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
760
|
-
|
761
|
-
You can choose the compute layer on the command line by executing e.g.
|
762
|
-
```
|
763
|
-
python myflow.py run --with batch
|
764
|
-
```
|
765
|
-
or
|
766
|
-
```
|
767
|
-
python myflow.py run --with kubernetes
|
768
|
-
```
|
769
|
-
which executes the flow on the desired system using the
|
770
|
-
requirements specified in `@resources`.
|
705
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
706
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
707
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
708
|
+
- `current.checkpoint`
|
709
|
+
- `current.model`
|
710
|
+
- `current.huggingface_hub`
|
771
711
|
|
712
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
713
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
714
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
772
715
|
|
773
|
-
|
774
|
-
|
775
|
-
cpu : int, default 1
|
776
|
-
Number of CPUs required for this step.
|
777
|
-
gpu : int, optional, default None
|
778
|
-
Number of GPUs required for this step.
|
779
|
-
disk : int, optional, default None
|
780
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
781
|
-
memory : int, default 4096
|
782
|
-
Memory size (in MB) required for this step.
|
783
|
-
shared_memory : int, optional, default None
|
784
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
785
|
-
This parameter maps to the `--shm-size` option in Docker.
|
786
|
-
"""
|
787
|
-
...
|
788
|
-
|
789
|
-
@typing.overload
|
790
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
791
|
-
"""
|
792
|
-
Decorator prototype for all step decorators. This function gets specialized
|
793
|
-
and imported for all decorators types by _import_plugin_decorators().
|
794
|
-
"""
|
795
|
-
...
|
796
|
-
|
797
|
-
@typing.overload
|
798
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
799
|
-
...
|
800
|
-
|
801
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
802
|
-
"""
|
803
|
-
Decorator prototype for all step decorators. This function gets specialized
|
804
|
-
and imported for all decorators types by _import_plugin_decorators().
|
716
|
+
temp_dir_root : str, default: None
|
717
|
+
The root directory under which `current.model.loaded` will store loaded models
|
805
718
|
"""
|
806
719
|
...
|
807
720
|
|
808
|
-
def
|
721
|
+
def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
809
722
|
"""
|
810
|
-
Specifies that this step
|
811
|
-
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
723
|
+
Specifies that this step should execute on DGX cloud.
|
812
724
|
|
813
725
|
|
814
726
|
Parameters
|
815
727
|
----------
|
816
|
-
|
728
|
+
gpu : int
|
817
729
|
Number of GPUs to use.
|
818
|
-
|
819
|
-
|
730
|
+
gpu_type : str
|
731
|
+
Type of Nvidia GPU to use.
|
820
732
|
"""
|
821
733
|
...
|
822
734
|
|
823
735
|
@typing.overload
|
824
|
-
def
|
736
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
825
737
|
"""
|
826
|
-
|
738
|
+
Specifies the Conda environment for the step.
|
827
739
|
|
740
|
+
Information in this decorator will augment any
|
741
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
742
|
+
you can use `@conda_base` to set packages required by all
|
743
|
+
steps and use `@conda` to specify step-specific overrides.
|
828
744
|
|
829
745
|
|
830
746
|
Parameters
|
831
747
|
----------
|
832
|
-
|
833
|
-
|
834
|
-
|
835
|
-
|
836
|
-
|
837
|
-
|
838
|
-
|
839
|
-
|
840
|
-
|
841
|
-
|
842
|
-
|
843
|
-
temp_dir_root : str, default: None
|
844
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
748
|
+
packages : Dict[str, str], default {}
|
749
|
+
Packages to use for this step. The key is the name of the package
|
750
|
+
and the value is the version to use.
|
751
|
+
libraries : Dict[str, str], default {}
|
752
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
753
|
+
python : str, optional, default None
|
754
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
755
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
756
|
+
disabled : bool, default False
|
757
|
+
If set to True, disables @conda.
|
845
758
|
"""
|
846
759
|
...
|
847
760
|
|
848
761
|
@typing.overload
|
849
|
-
def
|
762
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
850
763
|
...
|
851
764
|
|
852
765
|
@typing.overload
|
853
|
-
def
|
766
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
854
767
|
...
|
855
768
|
|
856
|
-
def
|
769
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
857
770
|
"""
|
858
|
-
|
771
|
+
Specifies the Conda environment for the step.
|
859
772
|
|
773
|
+
Information in this decorator will augment any
|
774
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
775
|
+
you can use `@conda_base` to set packages required by all
|
776
|
+
steps and use `@conda` to specify step-specific overrides.
|
860
777
|
|
861
778
|
|
862
779
|
Parameters
|
863
780
|
----------
|
864
|
-
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
|
875
|
-
temp_dir_root : str, default: None
|
876
|
-
The root directory under which `current.checkpoint.directory` will be created.
|
781
|
+
packages : Dict[str, str], default {}
|
782
|
+
Packages to use for this step. The key is the name of the package
|
783
|
+
and the value is the version to use.
|
784
|
+
libraries : Dict[str, str], default {}
|
785
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
786
|
+
python : str, optional, default None
|
787
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
788
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
789
|
+
disabled : bool, default False
|
790
|
+
If set to True, disables @conda.
|
877
791
|
"""
|
878
792
|
...
|
879
793
|
|
@@ -962,6 +876,217 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
962
876
|
"""
|
963
877
|
...
|
964
878
|
|
879
|
+
@typing.overload
|
880
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
881
|
+
"""
|
882
|
+
Internal decorator to support Fast bakery
|
883
|
+
"""
|
884
|
+
...
|
885
|
+
|
886
|
+
@typing.overload
|
887
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
888
|
+
...
|
889
|
+
|
890
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
891
|
+
"""
|
892
|
+
Internal decorator to support Fast bakery
|
893
|
+
"""
|
894
|
+
...
|
895
|
+
|
896
|
+
@typing.overload
|
897
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
898
|
+
"""
|
899
|
+
Enables checkpointing for a step.
|
900
|
+
|
901
|
+
|
902
|
+
|
903
|
+
Parameters
|
904
|
+
----------
|
905
|
+
load_policy : str, default: "fresh"
|
906
|
+
The policy for loading the checkpoint. The following policies are supported:
|
907
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
908
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
909
|
+
will be loaded at the start of the task.
|
910
|
+
- "none": Do not load any checkpoint
|
911
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
912
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
913
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
914
|
+
created within the task will be loaded when the task is retries execution on failure.
|
915
|
+
|
916
|
+
temp_dir_root : str, default: None
|
917
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
918
|
+
"""
|
919
|
+
...
|
920
|
+
|
921
|
+
@typing.overload
|
922
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
923
|
+
...
|
924
|
+
|
925
|
+
@typing.overload
|
926
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
927
|
+
...
|
928
|
+
|
929
|
+
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
930
|
+
"""
|
931
|
+
Enables checkpointing for a step.
|
932
|
+
|
933
|
+
|
934
|
+
|
935
|
+
Parameters
|
936
|
+
----------
|
937
|
+
load_policy : str, default: "fresh"
|
938
|
+
The policy for loading the checkpoint. The following policies are supported:
|
939
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
940
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
941
|
+
will be loaded at the start of the task.
|
942
|
+
- "none": Do not load any checkpoint
|
943
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
944
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
945
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
946
|
+
created within the task will be loaded when the task is retries execution on failure.
|
947
|
+
|
948
|
+
temp_dir_root : str, default: None
|
949
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
950
|
+
"""
|
951
|
+
...
|
952
|
+
|
953
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
954
|
+
"""
|
955
|
+
Specifies that this step should execute on DGX cloud.
|
956
|
+
|
957
|
+
|
958
|
+
Parameters
|
959
|
+
----------
|
960
|
+
gpu : int
|
961
|
+
Number of GPUs to use.
|
962
|
+
gpu_type : str
|
963
|
+
Type of Nvidia GPU to use.
|
964
|
+
queue_timeout : int
|
965
|
+
Time to keep the job in NVCF's queue.
|
966
|
+
"""
|
967
|
+
...
|
968
|
+
|
969
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
970
|
+
"""
|
971
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
972
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
973
|
+
|
974
|
+
|
975
|
+
Parameters
|
976
|
+
----------
|
977
|
+
timeout : int
|
978
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
979
|
+
poke_interval : int
|
980
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
981
|
+
mode : str
|
982
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
983
|
+
exponential_backoff : bool
|
984
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
985
|
+
pool : str
|
986
|
+
the slot pool this task should run in,
|
987
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
988
|
+
soft_fail : bool
|
989
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
990
|
+
name : str
|
991
|
+
Name of the sensor on Airflow
|
992
|
+
description : str
|
993
|
+
Description of sensor in the Airflow UI
|
994
|
+
external_dag_id : str
|
995
|
+
The dag_id that contains the task you want to wait for.
|
996
|
+
external_task_ids : List[str]
|
997
|
+
The list of task_ids that you want to wait for.
|
998
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
999
|
+
allowed_states : List[str]
|
1000
|
+
Iterable of allowed states, (Default: ['success'])
|
1001
|
+
failed_states : List[str]
|
1002
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1003
|
+
execution_delta : datetime.timedelta
|
1004
|
+
time difference with the previous execution to look at,
|
1005
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1006
|
+
check_existence: bool
|
1007
|
+
Set to True to check if the external task exists or check if
|
1008
|
+
the DAG to wait for exists. (Default: True)
|
1009
|
+
"""
|
1010
|
+
...
|
1011
|
+
|
1012
|
+
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1013
|
+
"""
|
1014
|
+
Specifies what flows belong to the same project.
|
1015
|
+
|
1016
|
+
A project-specific namespace is created for all flows that
|
1017
|
+
use the same `@project(name)`.
|
1018
|
+
|
1019
|
+
|
1020
|
+
Parameters
|
1021
|
+
----------
|
1022
|
+
name : str
|
1023
|
+
Project name. Make sure that the name is unique amongst all
|
1024
|
+
projects that use the same production scheduler. The name may
|
1025
|
+
contain only lowercase alphanumeric characters and underscores.
|
1026
|
+
|
1027
|
+
branch : Optional[str], default None
|
1028
|
+
The branch to use. If not specified, the branch is set to
|
1029
|
+
`user.<username>` unless `production` is set to `True`. This can
|
1030
|
+
also be set on the command line using `--branch` as a top-level option.
|
1031
|
+
It is an error to specify `branch` in the decorator and on the command line.
|
1032
|
+
|
1033
|
+
production : bool, default False
|
1034
|
+
Whether or not the branch is the production branch. This can also be set on the
|
1035
|
+
command line using `--production` as a top-level option. It is an error to specify
|
1036
|
+
`production` in the decorator and on the command line.
|
1037
|
+
The project branch name will be:
|
1038
|
+
- if `branch` is specified:
|
1039
|
+
- if `production` is True: `prod.<branch>`
|
1040
|
+
- if `production` is False: `test.<branch>`
|
1041
|
+
- if `branch` is not specified:
|
1042
|
+
- if `production` is True: `prod`
|
1043
|
+
- if `production` is False: `user.<username>`
|
1044
|
+
"""
|
1045
|
+
...
|
1046
|
+
|
1047
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1048
|
+
"""
|
1049
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1050
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1051
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1052
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1053
|
+
starts only after all sensors finish.
|
1054
|
+
|
1055
|
+
|
1056
|
+
Parameters
|
1057
|
+
----------
|
1058
|
+
timeout : int
|
1059
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1060
|
+
poke_interval : int
|
1061
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1062
|
+
mode : str
|
1063
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1064
|
+
exponential_backoff : bool
|
1065
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1066
|
+
pool : str
|
1067
|
+
the slot pool this task should run in,
|
1068
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1069
|
+
soft_fail : bool
|
1070
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1071
|
+
name : str
|
1072
|
+
Name of the sensor on Airflow
|
1073
|
+
description : str
|
1074
|
+
Description of sensor in the Airflow UI
|
1075
|
+
bucket_key : Union[str, List[str]]
|
1076
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1077
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1078
|
+
bucket_name : str
|
1079
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1080
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1081
|
+
wildcard_match : bool
|
1082
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1083
|
+
aws_conn_id : str
|
1084
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1085
|
+
verify : bool
|
1086
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1087
|
+
"""
|
1088
|
+
...
|
1089
|
+
|
965
1090
|
@typing.overload
|
966
1091
|
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
967
1092
|
"""
|
@@ -1055,6 +1180,57 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1055
1180
|
"""
|
1056
1181
|
...
|
1057
1182
|
|
1183
|
+
@typing.overload
|
1184
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1185
|
+
"""
|
1186
|
+
Specifies the Conda environment for all steps of the flow.
|
1187
|
+
|
1188
|
+
Use `@conda_base` to set common libraries required by all
|
1189
|
+
steps and use `@conda` to specify step-specific additions.
|
1190
|
+
|
1191
|
+
|
1192
|
+
Parameters
|
1193
|
+
----------
|
1194
|
+
packages : Dict[str, str], default {}
|
1195
|
+
Packages to use for this flow. The key is the name of the package
|
1196
|
+
and the value is the version to use.
|
1197
|
+
libraries : Dict[str, str], default {}
|
1198
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1199
|
+
python : str, optional, default None
|
1200
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1201
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1202
|
+
disabled : bool, default False
|
1203
|
+
If set to True, disables Conda.
|
1204
|
+
"""
|
1205
|
+
...
|
1206
|
+
|
1207
|
+
@typing.overload
|
1208
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1209
|
+
...
|
1210
|
+
|
1211
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1212
|
+
"""
|
1213
|
+
Specifies the Conda environment for all steps of the flow.
|
1214
|
+
|
1215
|
+
Use `@conda_base` to set common libraries required by all
|
1216
|
+
steps and use `@conda` to specify step-specific additions.
|
1217
|
+
|
1218
|
+
|
1219
|
+
Parameters
|
1220
|
+
----------
|
1221
|
+
packages : Dict[str, str], default {}
|
1222
|
+
Packages to use for this flow. The key is the name of the package
|
1223
|
+
and the value is the version to use.
|
1224
|
+
libraries : Dict[str, str], default {}
|
1225
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1226
|
+
python : str, optional, default None
|
1227
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1228
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1229
|
+
disabled : bool, default False
|
1230
|
+
If set to True, disables Conda.
|
1231
|
+
"""
|
1232
|
+
...
|
1233
|
+
|
1058
1234
|
def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
|
1059
1235
|
"""
|
1060
1236
|
Allows setting external datastores to save data for the
|
@@ -1169,49 +1345,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
|
|
1169
1345
|
"""
|
1170
1346
|
...
|
1171
1347
|
|
1172
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1173
|
-
"""
|
1174
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1175
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1176
|
-
|
1177
|
-
|
1178
|
-
Parameters
|
1179
|
-
----------
|
1180
|
-
timeout : int
|
1181
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1182
|
-
poke_interval : int
|
1183
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1184
|
-
mode : str
|
1185
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1186
|
-
exponential_backoff : bool
|
1187
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1188
|
-
pool : str
|
1189
|
-
the slot pool this task should run in,
|
1190
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1191
|
-
soft_fail : bool
|
1192
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1193
|
-
name : str
|
1194
|
-
Name of the sensor on Airflow
|
1195
|
-
description : str
|
1196
|
-
Description of sensor in the Airflow UI
|
1197
|
-
external_dag_id : str
|
1198
|
-
The dag_id that contains the task you want to wait for.
|
1199
|
-
external_task_ids : List[str]
|
1200
|
-
The list of task_ids that you want to wait for.
|
1201
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
1202
|
-
allowed_states : List[str]
|
1203
|
-
Iterable of allowed states, (Default: ['success'])
|
1204
|
-
failed_states : List[str]
|
1205
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
1206
|
-
execution_delta : datetime.timedelta
|
1207
|
-
time difference with the previous execution to look at,
|
1208
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
1209
|
-
check_existence: bool
|
1210
|
-
Set to True to check if the external task exists or check if
|
1211
|
-
the DAG to wait for exists. (Default: True)
|
1212
|
-
"""
|
1213
|
-
...
|
1214
|
-
|
1215
1348
|
@typing.overload
|
1216
1349
|
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1217
1350
|
"""
|
@@ -1253,49 +1386,6 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
|
|
1253
1386
|
"""
|
1254
1387
|
...
|
1255
1388
|
|
1256
|
-
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1257
|
-
"""
|
1258
|
-
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1259
|
-
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1260
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1261
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1262
|
-
starts only after all sensors finish.
|
1263
|
-
|
1264
|
-
|
1265
|
-
Parameters
|
1266
|
-
----------
|
1267
|
-
timeout : int
|
1268
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1269
|
-
poke_interval : int
|
1270
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1271
|
-
mode : str
|
1272
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1273
|
-
exponential_backoff : bool
|
1274
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1275
|
-
pool : str
|
1276
|
-
the slot pool this task should run in,
|
1277
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1278
|
-
soft_fail : bool
|
1279
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1280
|
-
name : str
|
1281
|
-
Name of the sensor on Airflow
|
1282
|
-
description : str
|
1283
|
-
Description of sensor in the Airflow UI
|
1284
|
-
bucket_key : Union[str, List[str]]
|
1285
|
-
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1286
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1287
|
-
bucket_name : str
|
1288
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1289
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1290
|
-
wildcard_match : bool
|
1291
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1292
|
-
aws_conn_id : str
|
1293
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
1294
|
-
verify : bool
|
1295
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1296
|
-
"""
|
1297
|
-
...
|
1298
|
-
|
1299
1389
|
@typing.overload
|
1300
1390
|
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1301
1391
|
"""
|
@@ -1448,91 +1538,5 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
|
|
1448
1538
|
"""
|
1449
1539
|
...
|
1450
1540
|
|
1451
|
-
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1452
|
-
"""
|
1453
|
-
Specifies what flows belong to the same project.
|
1454
|
-
|
1455
|
-
A project-specific namespace is created for all flows that
|
1456
|
-
use the same `@project(name)`.
|
1457
|
-
|
1458
|
-
|
1459
|
-
Parameters
|
1460
|
-
----------
|
1461
|
-
name : str
|
1462
|
-
Project name. Make sure that the name is unique amongst all
|
1463
|
-
projects that use the same production scheduler. The name may
|
1464
|
-
contain only lowercase alphanumeric characters and underscores.
|
1465
|
-
|
1466
|
-
branch : Optional[str], default None
|
1467
|
-
The branch to use. If not specified, the branch is set to
|
1468
|
-
`user.<username>` unless `production` is set to `True`. This can
|
1469
|
-
also be set on the command line using `--branch` as a top-level option.
|
1470
|
-
It is an error to specify `branch` in the decorator and on the command line.
|
1471
|
-
|
1472
|
-
production : bool, default False
|
1473
|
-
Whether or not the branch is the production branch. This can also be set on the
|
1474
|
-
command line using `--production` as a top-level option. It is an error to specify
|
1475
|
-
`production` in the decorator and on the command line.
|
1476
|
-
The project branch name will be:
|
1477
|
-
- if `branch` is specified:
|
1478
|
-
- if `production` is True: `prod.<branch>`
|
1479
|
-
- if `production` is False: `test.<branch>`
|
1480
|
-
- if `branch` is not specified:
|
1481
|
-
- if `production` is True: `prod`
|
1482
|
-
- if `production` is False: `user.<username>`
|
1483
|
-
"""
|
1484
|
-
...
|
1485
|
-
|
1486
|
-
@typing.overload
|
1487
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1488
|
-
"""
|
1489
|
-
Specifies the Conda environment for all steps of the flow.
|
1490
|
-
|
1491
|
-
Use `@conda_base` to set common libraries required by all
|
1492
|
-
steps and use `@conda` to specify step-specific additions.
|
1493
|
-
|
1494
|
-
|
1495
|
-
Parameters
|
1496
|
-
----------
|
1497
|
-
packages : Dict[str, str], default {}
|
1498
|
-
Packages to use for this flow. The key is the name of the package
|
1499
|
-
and the value is the version to use.
|
1500
|
-
libraries : Dict[str, str], default {}
|
1501
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1502
|
-
python : str, optional, default None
|
1503
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1504
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1505
|
-
disabled : bool, default False
|
1506
|
-
If set to True, disables Conda.
|
1507
|
-
"""
|
1508
|
-
...
|
1509
|
-
|
1510
|
-
@typing.overload
|
1511
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1512
|
-
...
|
1513
|
-
|
1514
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1515
|
-
"""
|
1516
|
-
Specifies the Conda environment for all steps of the flow.
|
1517
|
-
|
1518
|
-
Use `@conda_base` to set common libraries required by all
|
1519
|
-
steps and use `@conda` to specify step-specific additions.
|
1520
|
-
|
1521
|
-
|
1522
|
-
Parameters
|
1523
|
-
----------
|
1524
|
-
packages : Dict[str, str], default {}
|
1525
|
-
Packages to use for this flow. The key is the name of the package
|
1526
|
-
and the value is the version to use.
|
1527
|
-
libraries : Dict[str, str], default {}
|
1528
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1529
|
-
python : str, optional, default None
|
1530
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1531
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1532
|
-
disabled : bool, default False
|
1533
|
-
If set to True, disables Conda.
|
1534
|
-
"""
|
1535
|
-
...
|
1536
|
-
|
1537
1541
|
pkg_name: str
|
1538
1542
|
|