ob-metaflow-stubs 6.0.3.162__py2.py3-none-any.whl → 6.0.3.163__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +615 -615
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/cli_components/__init__.pyi +2 -2
- metaflow-stubs/cli_components/utils.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +3 -3
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +93 -93
- metaflow-stubs/metaflow_git.pyi +2 -2
- metaflow-stubs/mf_extensions/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +11 -11
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +5 -5
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +5 -5
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +5 -5
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
- metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
- metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/plugins/uv/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +29 -29
- metaflow-stubs/runner/deployer_impl.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +3 -3
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +3 -3
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/user_configs/__init__.pyi +2 -2
- metaflow-stubs/user_configs/config_decorators.pyi +6 -6
- metaflow-stubs/user_configs/config_options.pyi +3 -3
- metaflow-stubs/user_configs/config_parameters.pyi +5 -5
- {ob_metaflow_stubs-6.0.3.162.dist-info → ob_metaflow_stubs-6.0.3.163.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.163.dist-info/RECORD +206 -0
- ob_metaflow_stubs-6.0.3.162.dist-info/RECORD +0 -206
- {ob_metaflow_stubs-6.0.3.162.dist-info → ob_metaflow_stubs-6.0.3.163.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.162.dist-info → ob_metaflow_stubs-6.0.3.163.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.15.
|
4
|
-
# Generated on 2025-05-
|
3
|
+
# MF version: 2.15.11.1+obcheckpoint(0.2.1);ob(v1) #
|
4
|
+
# Generated on 2025-05-06T23:22:15.553269 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import datetime
|
12
11
|
import typing
|
12
|
+
import datetime
|
13
13
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
14
14
|
StepFlag = typing.NewType("StepFlag", bool)
|
15
15
|
|
@@ -37,16 +37,16 @@ from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDec
|
|
37
37
|
from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
|
38
38
|
from . import cards as cards
|
39
39
|
from . import tuple_util as tuple_util
|
40
|
-
from . import metaflow_git as metaflow_git
|
41
40
|
from . import events as events
|
41
|
+
from . import metaflow_git as metaflow_git
|
42
42
|
from . import runner as runner
|
43
43
|
from . import plugins as plugins
|
44
44
|
from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
|
45
45
|
from . import includefile as includefile
|
46
46
|
from .includefile import IncludeFile as IncludeFile
|
47
|
+
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
47
48
|
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
48
49
|
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
49
|
-
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
50
50
|
from . import client as client
|
51
51
|
from .client.core import namespace as namespace
|
52
52
|
from .client.core import get_namespace as get_namespace
|
@@ -153,18 +153,87 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
153
153
|
"""
|
154
154
|
...
|
155
155
|
|
156
|
-
|
156
|
+
@typing.overload
|
157
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
157
158
|
"""
|
158
|
-
Specifies that
|
159
|
-
|
159
|
+
Specifies that the step will success under all circumstances.
|
160
|
+
|
161
|
+
The decorator will create an optional artifact, specified by `var`, which
|
162
|
+
contains the exception raised. You can use it to detect the presence
|
163
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
164
|
+
are missing.
|
160
165
|
|
161
166
|
|
162
167
|
Parameters
|
163
168
|
----------
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
169
|
+
var : str, optional, default None
|
170
|
+
Name of the artifact in which to store the caught exception.
|
171
|
+
If not specified, the exception is not stored.
|
172
|
+
print_exception : bool, default True
|
173
|
+
Determines whether or not the exception is printed to
|
174
|
+
stdout when caught.
|
175
|
+
"""
|
176
|
+
...
|
177
|
+
|
178
|
+
@typing.overload
|
179
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
180
|
+
...
|
181
|
+
|
182
|
+
@typing.overload
|
183
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
184
|
+
...
|
185
|
+
|
186
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
187
|
+
"""
|
188
|
+
Specifies that the step will success under all circumstances.
|
189
|
+
|
190
|
+
The decorator will create an optional artifact, specified by `var`, which
|
191
|
+
contains the exception raised. You can use it to detect the presence
|
192
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
193
|
+
are missing.
|
194
|
+
|
195
|
+
|
196
|
+
Parameters
|
197
|
+
----------
|
198
|
+
var : str, optional, default None
|
199
|
+
Name of the artifact in which to store the caught exception.
|
200
|
+
If not specified, the exception is not stored.
|
201
|
+
print_exception : bool, default True
|
202
|
+
Determines whether or not the exception is printed to
|
203
|
+
stdout when caught.
|
204
|
+
"""
|
205
|
+
...
|
206
|
+
|
207
|
+
@typing.overload
|
208
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
209
|
+
"""
|
210
|
+
Specifies environment variables to be set prior to the execution of a step.
|
211
|
+
|
212
|
+
|
213
|
+
Parameters
|
214
|
+
----------
|
215
|
+
vars : Dict[str, str], default {}
|
216
|
+
Dictionary of environment variables to set.
|
217
|
+
"""
|
218
|
+
...
|
219
|
+
|
220
|
+
@typing.overload
|
221
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
222
|
+
...
|
223
|
+
|
224
|
+
@typing.overload
|
225
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
226
|
+
...
|
227
|
+
|
228
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
229
|
+
"""
|
230
|
+
Specifies environment variables to be set prior to the execution of a step.
|
231
|
+
|
232
|
+
|
233
|
+
Parameters
|
234
|
+
----------
|
235
|
+
vars : Dict[str, str], default {}
|
236
|
+
Dictionary of environment variables to set.
|
168
237
|
"""
|
169
238
|
...
|
170
239
|
|
@@ -194,68 +263,105 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
|
|
194
263
|
...
|
195
264
|
|
196
265
|
@typing.overload
|
197
|
-
def
|
266
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
198
267
|
"""
|
199
|
-
|
268
|
+
Enables loading / saving of models within a step.
|
269
|
+
|
270
|
+
|
271
|
+
|
272
|
+
Parameters
|
273
|
+
----------
|
274
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
275
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
276
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
277
|
+
- `current.checkpoint`
|
278
|
+
- `current.model`
|
279
|
+
- `current.huggingface_hub`
|
280
|
+
|
281
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
282
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
283
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
284
|
+
|
285
|
+
temp_dir_root : str, default: None
|
286
|
+
The root directory under which `current.model.loaded` will store loaded models
|
200
287
|
"""
|
201
288
|
...
|
202
289
|
|
203
290
|
@typing.overload
|
204
|
-
def
|
291
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
205
292
|
...
|
206
293
|
|
207
|
-
|
208
|
-
|
209
|
-
Internal decorator to support Fast bakery
|
210
|
-
"""
|
294
|
+
@typing.overload
|
295
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
211
296
|
...
|
212
297
|
|
213
|
-
|
214
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
298
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
215
299
|
"""
|
216
|
-
|
217
|
-
|
300
|
+
Enables loading / saving of models within a step.
|
301
|
+
|
218
302
|
|
219
303
|
|
220
304
|
Parameters
|
221
305
|
----------
|
222
|
-
|
223
|
-
|
306
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
307
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
308
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
309
|
+
- `current.checkpoint`
|
310
|
+
- `current.model`
|
311
|
+
- `current.huggingface_hub`
|
312
|
+
|
313
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
314
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
315
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
316
|
+
|
317
|
+
temp_dir_root : str, default: None
|
318
|
+
The root directory under which `current.model.loaded` will store loaded models
|
224
319
|
"""
|
225
320
|
...
|
226
321
|
|
227
|
-
|
228
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
229
|
-
...
|
230
|
-
|
231
|
-
@typing.overload
|
232
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
233
|
-
...
|
234
|
-
|
235
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
322
|
+
def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
236
323
|
"""
|
237
|
-
Specifies
|
238
|
-
|
324
|
+
Specifies that this step is used to deploy an instance of the app.
|
325
|
+
Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
|
239
326
|
|
240
327
|
|
241
328
|
Parameters
|
242
329
|
----------
|
243
|
-
|
244
|
-
|
330
|
+
app_port : int
|
331
|
+
Number of GPUs to use.
|
332
|
+
app_name : str
|
333
|
+
Name of the app to deploy.
|
245
334
|
"""
|
246
335
|
...
|
247
336
|
|
248
|
-
def
|
337
|
+
def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
249
338
|
"""
|
250
|
-
|
339
|
+
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
340
|
+
|
341
|
+
User code call
|
342
|
+
-----------
|
343
|
+
@nim(
|
344
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
345
|
+
backend='managed'
|
346
|
+
)
|
347
|
+
|
348
|
+
Valid backend options
|
349
|
+
---------------------
|
350
|
+
- 'managed': Outerbounds selects a compute provider based on the model.
|
351
|
+
|
352
|
+
Valid model options
|
353
|
+
----------------
|
354
|
+
- 'meta/llama3-8b-instruct': 8B parameter model
|
355
|
+
- 'meta/llama3-70b-instruct': 70B parameter model
|
356
|
+
- any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
251
357
|
|
252
358
|
|
253
359
|
Parameters
|
254
360
|
----------
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
361
|
+
models: list[NIM]
|
362
|
+
List of NIM containers running models in sidecars.
|
363
|
+
backend: str
|
364
|
+
Compute provider to run the NIM container.
|
259
365
|
queue_timeout : int
|
260
366
|
Time to keep the job in NVCF's queue.
|
261
367
|
"""
|
@@ -380,91 +486,154 @@ def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.U
|
|
380
486
|
...
|
381
487
|
|
382
488
|
@typing.overload
|
383
|
-
def
|
489
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
384
490
|
"""
|
385
|
-
|
386
|
-
and imported for all decorators types by _import_plugin_decorators().
|
491
|
+
Internal decorator to support Fast bakery
|
387
492
|
"""
|
388
493
|
...
|
389
494
|
|
390
495
|
@typing.overload
|
391
|
-
def
|
496
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
392
497
|
...
|
393
498
|
|
394
|
-
def
|
499
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
395
500
|
"""
|
396
|
-
|
397
|
-
and imported for all decorators types by _import_plugin_decorators().
|
501
|
+
Internal decorator to support Fast bakery
|
398
502
|
"""
|
399
503
|
...
|
400
504
|
|
401
|
-
|
505
|
+
@typing.overload
|
506
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
402
507
|
"""
|
403
|
-
|
404
|
-
|
405
|
-
User code call
|
406
|
-
-----------
|
407
|
-
@nim(
|
408
|
-
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
409
|
-
backend='managed'
|
410
|
-
)
|
411
|
-
|
412
|
-
Valid backend options
|
413
|
-
---------------------
|
414
|
-
- 'managed': Outerbounds selects a compute provider based on the model.
|
508
|
+
Specifies the PyPI packages for the step.
|
415
509
|
|
416
|
-
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
- any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
510
|
+
Information in this decorator will augment any
|
511
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
512
|
+
you can use `@pypi_base` to set packages required by all
|
513
|
+
steps and use `@pypi` to specify step-specific overrides.
|
421
514
|
|
422
515
|
|
423
516
|
Parameters
|
424
517
|
----------
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
518
|
+
packages : Dict[str, str], default: {}
|
519
|
+
Packages to use for this step. The key is the name of the package
|
520
|
+
and the value is the version to use.
|
521
|
+
python : str, optional, default: None
|
522
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
523
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
431
524
|
"""
|
432
525
|
...
|
433
526
|
|
434
527
|
@typing.overload
|
435
|
-
def
|
528
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
529
|
+
...
|
530
|
+
|
531
|
+
@typing.overload
|
532
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
533
|
+
...
|
534
|
+
|
535
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
436
536
|
"""
|
437
|
-
|
537
|
+
Specifies the PyPI packages for the step.
|
438
538
|
|
539
|
+
Information in this decorator will augment any
|
540
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
541
|
+
you can use `@pypi_base` to set packages required by all
|
542
|
+
steps and use `@pypi` to specify step-specific overrides.
|
439
543
|
|
440
544
|
|
441
545
|
Parameters
|
442
546
|
----------
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
547
|
+
packages : Dict[str, str], default: {}
|
548
|
+
Packages to use for this step. The key is the name of the package
|
549
|
+
and the value is the version to use.
|
550
|
+
python : str, optional, default: None
|
551
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
552
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
553
|
+
"""
|
554
|
+
...
|
555
|
+
|
556
|
+
@typing.overload
|
557
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
558
|
+
"""
|
559
|
+
Specifies the resources needed when executing this step.
|
453
560
|
|
454
|
-
|
455
|
-
|
561
|
+
Use `@resources` to specify the resource requirements
|
562
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
563
|
+
|
564
|
+
You can choose the compute layer on the command line by executing e.g.
|
565
|
+
```
|
566
|
+
python myflow.py run --with batch
|
567
|
+
```
|
568
|
+
or
|
569
|
+
```
|
570
|
+
python myflow.py run --with kubernetes
|
571
|
+
```
|
572
|
+
which executes the flow on the desired system using the
|
573
|
+
requirements specified in `@resources`.
|
574
|
+
|
575
|
+
|
576
|
+
Parameters
|
577
|
+
----------
|
578
|
+
cpu : int, default 1
|
579
|
+
Number of CPUs required for this step.
|
580
|
+
gpu : int, optional, default None
|
581
|
+
Number of GPUs required for this step.
|
582
|
+
disk : int, optional, default None
|
583
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
584
|
+
memory : int, default 4096
|
585
|
+
Memory size (in MB) required for this step.
|
586
|
+
shared_memory : int, optional, default None
|
587
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
588
|
+
This parameter maps to the `--shm-size` option in Docker.
|
456
589
|
"""
|
457
590
|
...
|
458
591
|
|
459
592
|
@typing.overload
|
460
|
-
def
|
593
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
461
594
|
...
|
462
595
|
|
463
596
|
@typing.overload
|
464
|
-
def
|
597
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
465
598
|
...
|
466
599
|
|
467
|
-
def
|
600
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
601
|
+
"""
|
602
|
+
Specifies the resources needed when executing this step.
|
603
|
+
|
604
|
+
Use `@resources` to specify the resource requirements
|
605
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
606
|
+
|
607
|
+
You can choose the compute layer on the command line by executing e.g.
|
608
|
+
```
|
609
|
+
python myflow.py run --with batch
|
610
|
+
```
|
611
|
+
or
|
612
|
+
```
|
613
|
+
python myflow.py run --with kubernetes
|
614
|
+
```
|
615
|
+
which executes the flow on the desired system using the
|
616
|
+
requirements specified in `@resources`.
|
617
|
+
|
618
|
+
|
619
|
+
Parameters
|
620
|
+
----------
|
621
|
+
cpu : int, default 1
|
622
|
+
Number of CPUs required for this step.
|
623
|
+
gpu : int, optional, default None
|
624
|
+
Number of GPUs required for this step.
|
625
|
+
disk : int, optional, default None
|
626
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
627
|
+
memory : int, default 4096
|
628
|
+
Memory size (in MB) required for this step.
|
629
|
+
shared_memory : int, optional, default None
|
630
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
631
|
+
This parameter maps to the `--shm-size` option in Docker.
|
632
|
+
"""
|
633
|
+
...
|
634
|
+
|
635
|
+
@typing.overload
|
636
|
+
def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
468
637
|
"""
|
469
638
|
Enables checkpointing for a step.
|
470
639
|
|
@@ -489,108 +658,53 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
|
|
489
658
|
...
|
490
659
|
|
491
660
|
@typing.overload
|
492
|
-
def
|
493
|
-
"""
|
494
|
-
Specifies that the step will success under all circumstances.
|
495
|
-
|
496
|
-
The decorator will create an optional artifact, specified by `var`, which
|
497
|
-
contains the exception raised. You can use it to detect the presence
|
498
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
499
|
-
are missing.
|
500
|
-
|
501
|
-
|
502
|
-
Parameters
|
503
|
-
----------
|
504
|
-
var : str, optional, default None
|
505
|
-
Name of the artifact in which to store the caught exception.
|
506
|
-
If not specified, the exception is not stored.
|
507
|
-
print_exception : bool, default True
|
508
|
-
Determines whether or not the exception is printed to
|
509
|
-
stdout when caught.
|
510
|
-
"""
|
511
|
-
...
|
512
|
-
|
513
|
-
@typing.overload
|
514
|
-
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
661
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
515
662
|
...
|
516
663
|
|
517
664
|
@typing.overload
|
518
|
-
def
|
665
|
+
def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
519
666
|
...
|
520
667
|
|
521
|
-
def
|
668
|
+
def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
|
522
669
|
"""
|
523
|
-
|
670
|
+
Enables checkpointing for a step.
|
524
671
|
|
525
|
-
The decorator will create an optional artifact, specified by `var`, which
|
526
|
-
contains the exception raised. You can use it to detect the presence
|
527
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
528
|
-
are missing.
|
529
672
|
|
530
673
|
|
531
674
|
Parameters
|
532
675
|
----------
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
676
|
+
load_policy : str, default: "fresh"
|
677
|
+
The policy for loading the checkpoint. The following policies are supported:
|
678
|
+
- "eager": Loads the the latest available checkpoint within the namespace.
|
679
|
+
With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
|
680
|
+
will be loaded at the start of the task.
|
681
|
+
- "none": Do not load any checkpoint
|
682
|
+
- "fresh": Loads the lastest checkpoint created within the running Task.
|
683
|
+
This mode helps loading checkpoints across various retry attempts of the same task.
|
684
|
+
With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
|
685
|
+
created within the task will be loaded when the task is retries execution on failure.
|
686
|
+
|
687
|
+
temp_dir_root : str, default: None
|
688
|
+
The root directory under which `current.checkpoint.directory` will be created.
|
539
689
|
"""
|
540
690
|
...
|
541
691
|
|
542
692
|
@typing.overload
|
543
|
-
def
|
693
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
544
694
|
"""
|
545
|
-
|
546
|
-
|
547
|
-
|
548
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
549
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
550
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
551
|
-
|
552
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
553
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
554
|
-
ensuring that the flow execution can continue.
|
555
|
-
|
556
|
-
|
557
|
-
Parameters
|
558
|
-
----------
|
559
|
-
times : int, default 3
|
560
|
-
Number of times to retry this task.
|
561
|
-
minutes_between_retries : int, default 2
|
562
|
-
Number of minutes between retries.
|
695
|
+
Decorator prototype for all step decorators. This function gets specialized
|
696
|
+
and imported for all decorators types by _import_plugin_decorators().
|
563
697
|
"""
|
564
698
|
...
|
565
699
|
|
566
700
|
@typing.overload
|
567
|
-
def
|
568
|
-
...
|
569
|
-
|
570
|
-
@typing.overload
|
571
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
701
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
572
702
|
...
|
573
703
|
|
574
|
-
def
|
704
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
575
705
|
"""
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
580
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
581
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
582
|
-
|
583
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
584
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
585
|
-
ensuring that the flow execution can continue.
|
586
|
-
|
587
|
-
|
588
|
-
Parameters
|
589
|
-
----------
|
590
|
-
times : int, default 3
|
591
|
-
Number of times to retry this task.
|
592
|
-
minutes_between_retries : int, default 2
|
593
|
-
Number of minutes between retries.
|
706
|
+
Decorator prototype for all step decorators. This function gets specialized
|
707
|
+
and imported for all decorators types by _import_plugin_decorators().
|
594
708
|
"""
|
595
709
|
...
|
596
710
|
|
@@ -702,317 +816,168 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
702
816
|
"""
|
703
817
|
...
|
704
818
|
|
705
|
-
|
706
|
-
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
819
|
+
def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
707
820
|
"""
|
708
|
-
Specifies
|
821
|
+
Specifies that this step should execute on DGX cloud.
|
709
822
|
|
710
|
-
This decorator is useful if this step may hang indefinitely.
|
711
823
|
|
712
|
-
|
713
|
-
|
714
|
-
|
824
|
+
Parameters
|
825
|
+
----------
|
826
|
+
gpu : int
|
827
|
+
Number of GPUs to use.
|
828
|
+
gpu_type : str
|
829
|
+
Type of Nvidia GPU to use.
|
830
|
+
queue_timeout : int
|
831
|
+
Time to keep the job in NVCF's queue.
|
832
|
+
"""
|
833
|
+
...
|
834
|
+
|
835
|
+
@typing.overload
|
836
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
837
|
+
"""
|
838
|
+
Specifies the number of times the task corresponding
|
839
|
+
to a step needs to be retried.
|
715
840
|
|
716
|
-
|
717
|
-
|
841
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
842
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
843
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
844
|
+
|
845
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
846
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
847
|
+
ensuring that the flow execution can continue.
|
718
848
|
|
719
849
|
|
720
850
|
Parameters
|
721
851
|
----------
|
722
|
-
|
723
|
-
Number of
|
724
|
-
|
725
|
-
Number of minutes
|
726
|
-
hours : int, default 0
|
727
|
-
Number of hours to wait prior to timing out.
|
852
|
+
times : int, default 3
|
853
|
+
Number of times to retry this task.
|
854
|
+
minutes_between_retries : int, default 2
|
855
|
+
Number of minutes between retries.
|
728
856
|
"""
|
729
857
|
...
|
730
858
|
|
731
859
|
@typing.overload
|
732
|
-
def
|
860
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
733
861
|
...
|
734
862
|
|
735
863
|
@typing.overload
|
736
|
-
def
|
864
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
737
865
|
...
|
738
866
|
|
739
|
-
def
|
867
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
740
868
|
"""
|
741
|
-
Specifies
|
869
|
+
Specifies the number of times the task corresponding
|
870
|
+
to a step needs to be retried.
|
742
871
|
|
743
|
-
This decorator is useful
|
872
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
873
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
874
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
744
875
|
|
745
|
-
This can be used in conjunction with the `@
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
Note that all the values specified in parameters are added together so if you specify
|
750
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
751
|
-
|
752
|
-
|
753
|
-
Parameters
|
754
|
-
----------
|
755
|
-
seconds : int, default 0
|
756
|
-
Number of seconds to wait prior to timing out.
|
757
|
-
minutes : int, default 0
|
758
|
-
Number of minutes to wait prior to timing out.
|
759
|
-
hours : int, default 0
|
760
|
-
Number of hours to wait prior to timing out.
|
761
|
-
"""
|
762
|
-
...
|
763
|
-
|
764
|
-
@typing.overload
|
765
|
-
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
766
|
-
"""
|
767
|
-
Specifies the resources needed when executing this step.
|
768
|
-
|
769
|
-
Use `@resources` to specify the resource requirements
|
770
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
771
|
-
|
772
|
-
You can choose the compute layer on the command line by executing e.g.
|
773
|
-
```
|
774
|
-
python myflow.py run --with batch
|
775
|
-
```
|
776
|
-
or
|
777
|
-
```
|
778
|
-
python myflow.py run --with kubernetes
|
779
|
-
```
|
780
|
-
which executes the flow on the desired system using the
|
781
|
-
requirements specified in `@resources`.
|
876
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
877
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
878
|
+
ensuring that the flow execution can continue.
|
782
879
|
|
783
880
|
|
784
881
|
Parameters
|
785
882
|
----------
|
786
|
-
|
787
|
-
Number of
|
788
|
-
|
789
|
-
Number of
|
790
|
-
disk : int, optional, default None
|
791
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
792
|
-
memory : int, default 4096
|
793
|
-
Memory size (in MB) required for this step.
|
794
|
-
shared_memory : int, optional, default None
|
795
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
796
|
-
This parameter maps to the `--shm-size` option in Docker.
|
883
|
+
times : int, default 3
|
884
|
+
Number of times to retry this task.
|
885
|
+
minutes_between_retries : int, default 2
|
886
|
+
Number of minutes between retries.
|
797
887
|
"""
|
798
888
|
...
|
799
889
|
|
800
890
|
@typing.overload
|
801
|
-
def
|
802
|
-
...
|
803
|
-
|
804
|
-
@typing.overload
|
805
|
-
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
806
|
-
...
|
807
|
-
|
808
|
-
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
891
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
809
892
|
"""
|
810
|
-
Specifies
|
811
|
-
|
812
|
-
Use `@resources` to specify the resource requirements
|
813
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
893
|
+
Specifies a timeout for your step.
|
814
894
|
|
815
|
-
|
816
|
-
```
|
817
|
-
python myflow.py run --with batch
|
818
|
-
```
|
819
|
-
or
|
820
|
-
```
|
821
|
-
python myflow.py run --with kubernetes
|
822
|
-
```
|
823
|
-
which executes the flow on the desired system using the
|
824
|
-
requirements specified in `@resources`.
|
895
|
+
This decorator is useful if this step may hang indefinitely.
|
825
896
|
|
897
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
898
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
899
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
826
900
|
|
827
|
-
|
828
|
-
|
829
|
-
cpu : int, default 1
|
830
|
-
Number of CPUs required for this step.
|
831
|
-
gpu : int, optional, default None
|
832
|
-
Number of GPUs required for this step.
|
833
|
-
disk : int, optional, default None
|
834
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
835
|
-
memory : int, default 4096
|
836
|
-
Memory size (in MB) required for this step.
|
837
|
-
shared_memory : int, optional, default None
|
838
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
839
|
-
This parameter maps to the `--shm-size` option in Docker.
|
840
|
-
"""
|
841
|
-
...
|
842
|
-
|
843
|
-
@typing.overload
|
844
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
845
|
-
"""
|
846
|
-
Specifies environment variables to be set prior to the execution of a step.
|
901
|
+
Note that all the values specified in parameters are added together so if you specify
|
902
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
847
903
|
|
848
904
|
|
849
905
|
Parameters
|
850
906
|
----------
|
851
|
-
|
852
|
-
|
907
|
+
seconds : int, default 0
|
908
|
+
Number of seconds to wait prior to timing out.
|
909
|
+
minutes : int, default 0
|
910
|
+
Number of minutes to wait prior to timing out.
|
911
|
+
hours : int, default 0
|
912
|
+
Number of hours to wait prior to timing out.
|
853
913
|
"""
|
854
914
|
...
|
855
915
|
|
856
916
|
@typing.overload
|
857
|
-
def
|
917
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
858
918
|
...
|
859
919
|
|
860
920
|
@typing.overload
|
861
|
-
def
|
862
|
-
...
|
863
|
-
|
864
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
865
|
-
"""
|
866
|
-
Specifies environment variables to be set prior to the execution of a step.
|
867
|
-
|
868
|
-
|
869
|
-
Parameters
|
870
|
-
----------
|
871
|
-
vars : Dict[str, str], default {}
|
872
|
-
Dictionary of environment variables to set.
|
873
|
-
"""
|
921
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
874
922
|
...
|
875
923
|
|
876
|
-
|
877
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
924
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
878
925
|
"""
|
879
|
-
Specifies
|
880
|
-
|
881
|
-
Information in this decorator will augment any
|
882
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
883
|
-
you can use `@pypi_base` to set packages required by all
|
884
|
-
steps and use `@pypi` to specify step-specific overrides.
|
926
|
+
Specifies a timeout for your step.
|
885
927
|
|
928
|
+
This decorator is useful if this step may hang indefinitely.
|
886
929
|
|
887
|
-
|
888
|
-
|
889
|
-
|
890
|
-
Packages to use for this step. The key is the name of the package
|
891
|
-
and the value is the version to use.
|
892
|
-
python : str, optional, default: None
|
893
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
894
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
895
|
-
"""
|
896
|
-
...
|
897
|
-
|
898
|
-
@typing.overload
|
899
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
900
|
-
...
|
901
|
-
|
902
|
-
@typing.overload
|
903
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
904
|
-
...
|
905
|
-
|
906
|
-
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
907
|
-
"""
|
908
|
-
Specifies the PyPI packages for the step.
|
930
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
931
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
932
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
909
933
|
|
910
|
-
|
911
|
-
|
912
|
-
you can use `@pypi_base` to set packages required by all
|
913
|
-
steps and use `@pypi` to specify step-specific overrides.
|
934
|
+
Note that all the values specified in parameters are added together so if you specify
|
935
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
914
936
|
|
915
937
|
|
916
938
|
Parameters
|
917
939
|
----------
|
918
|
-
|
919
|
-
|
920
|
-
|
921
|
-
|
922
|
-
|
923
|
-
|
940
|
+
seconds : int, default 0
|
941
|
+
Number of seconds to wait prior to timing out.
|
942
|
+
minutes : int, default 0
|
943
|
+
Number of minutes to wait prior to timing out.
|
944
|
+
hours : int, default 0
|
945
|
+
Number of hours to wait prior to timing out.
|
924
946
|
"""
|
925
947
|
...
|
926
948
|
|
927
949
|
@typing.overload
|
928
|
-
def
|
950
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
929
951
|
"""
|
930
|
-
|
931
|
-
|
952
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
953
|
+
the execution of a step.
|
932
954
|
|
933
955
|
|
934
956
|
Parameters
|
935
957
|
----------
|
936
|
-
|
937
|
-
|
938
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
939
|
-
- `current.checkpoint`
|
940
|
-
- `current.model`
|
941
|
-
- `current.huggingface_hub`
|
942
|
-
|
943
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
944
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
945
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
946
|
-
|
947
|
-
temp_dir_root : str, default: None
|
948
|
-
The root directory under which `current.model.loaded` will store loaded models
|
958
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
959
|
+
List of secret specs, defining how the secrets are to be retrieved
|
949
960
|
"""
|
950
961
|
...
|
951
962
|
|
952
963
|
@typing.overload
|
953
|
-
def
|
964
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
954
965
|
...
|
955
966
|
|
956
967
|
@typing.overload
|
957
|
-
def
|
958
|
-
...
|
959
|
-
|
960
|
-
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
961
|
-
"""
|
962
|
-
Enables loading / saving of models within a step.
|
963
|
-
|
964
|
-
|
965
|
-
|
966
|
-
Parameters
|
967
|
-
----------
|
968
|
-
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
969
|
-
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
970
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
971
|
-
- `current.checkpoint`
|
972
|
-
- `current.model`
|
973
|
-
- `current.huggingface_hub`
|
974
|
-
|
975
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
976
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
977
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
978
|
-
|
979
|
-
temp_dir_root : str, default: None
|
980
|
-
The root directory under which `current.model.loaded` will store loaded models
|
981
|
-
"""
|
968
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
982
969
|
...
|
983
970
|
|
984
|
-
def
|
971
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
985
972
|
"""
|
986
|
-
Specifies
|
987
|
-
|
988
|
-
A project-specific namespace is created for all flows that
|
989
|
-
use the same `@project(name)`.
|
973
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
974
|
+
the execution of a step.
|
990
975
|
|
991
976
|
|
992
977
|
Parameters
|
993
978
|
----------
|
994
|
-
|
995
|
-
|
996
|
-
projects that use the same production scheduler. The name may
|
997
|
-
contain only lowercase alphanumeric characters and underscores.
|
998
|
-
|
999
|
-
branch : Optional[str], default None
|
1000
|
-
The branch to use. If not specified, the branch is set to
|
1001
|
-
`user.<username>` unless `production` is set to `True`. This can
|
1002
|
-
also be set on the command line using `--branch` as a top-level option.
|
1003
|
-
It is an error to specify `branch` in the decorator and on the command line.
|
1004
|
-
|
1005
|
-
production : bool, default False
|
1006
|
-
Whether or not the branch is the production branch. This can also be set on the
|
1007
|
-
command line using `--production` as a top-level option. It is an error to specify
|
1008
|
-
`production` in the decorator and on the command line.
|
1009
|
-
The project branch name will be:
|
1010
|
-
- if `branch` is specified:
|
1011
|
-
- if `production` is True: `prod.<branch>`
|
1012
|
-
- if `production` is False: `test.<branch>`
|
1013
|
-
- if `branch` is not specified:
|
1014
|
-
- if `production` is True: `prod`
|
1015
|
-
- if `production` is False: `user.<username>`
|
979
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
980
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1016
981
|
"""
|
1017
982
|
...
|
1018
983
|
|
@@ -1059,107 +1024,6 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1059
1024
|
"""
|
1060
1025
|
...
|
1061
1026
|
|
1062
|
-
@typing.overload
|
1063
|
-
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1064
|
-
"""
|
1065
|
-
Specifies the flow(s) that this flow depends on.
|
1066
|
-
|
1067
|
-
```
|
1068
|
-
@trigger_on_finish(flow='FooFlow')
|
1069
|
-
```
|
1070
|
-
or
|
1071
|
-
```
|
1072
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1073
|
-
```
|
1074
|
-
This decorator respects the @project decorator and triggers the flow
|
1075
|
-
when upstream runs within the same namespace complete successfully
|
1076
|
-
|
1077
|
-
Additionally, you can specify project aware upstream flow dependencies
|
1078
|
-
by specifying the fully qualified project_flow_name.
|
1079
|
-
```
|
1080
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1081
|
-
```
|
1082
|
-
or
|
1083
|
-
```
|
1084
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1085
|
-
```
|
1086
|
-
|
1087
|
-
You can also specify just the project or project branch (other values will be
|
1088
|
-
inferred from the current project or project branch):
|
1089
|
-
```
|
1090
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1091
|
-
```
|
1092
|
-
|
1093
|
-
Note that `branch` is typically one of:
|
1094
|
-
- `prod`
|
1095
|
-
- `user.bob`
|
1096
|
-
- `test.my_experiment`
|
1097
|
-
- `prod.staging`
|
1098
|
-
|
1099
|
-
|
1100
|
-
Parameters
|
1101
|
-
----------
|
1102
|
-
flow : Union[str, Dict[str, str]], optional, default None
|
1103
|
-
Upstream flow dependency for this flow.
|
1104
|
-
flows : List[Union[str, Dict[str, str]]], default []
|
1105
|
-
Upstream flow dependencies for this flow.
|
1106
|
-
options : Dict[str, Any], default {}
|
1107
|
-
Backend-specific configuration for tuning eventing behavior.
|
1108
|
-
"""
|
1109
|
-
...
|
1110
|
-
|
1111
|
-
@typing.overload
|
1112
|
-
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1113
|
-
...
|
1114
|
-
|
1115
|
-
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1116
|
-
"""
|
1117
|
-
Specifies the flow(s) that this flow depends on.
|
1118
|
-
|
1119
|
-
```
|
1120
|
-
@trigger_on_finish(flow='FooFlow')
|
1121
|
-
```
|
1122
|
-
or
|
1123
|
-
```
|
1124
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1125
|
-
```
|
1126
|
-
This decorator respects the @project decorator and triggers the flow
|
1127
|
-
when upstream runs within the same namespace complete successfully
|
1128
|
-
|
1129
|
-
Additionally, you can specify project aware upstream flow dependencies
|
1130
|
-
by specifying the fully qualified project_flow_name.
|
1131
|
-
```
|
1132
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1133
|
-
```
|
1134
|
-
or
|
1135
|
-
```
|
1136
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1137
|
-
```
|
1138
|
-
|
1139
|
-
You can also specify just the project or project branch (other values will be
|
1140
|
-
inferred from the current project or project branch):
|
1141
|
-
```
|
1142
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1143
|
-
```
|
1144
|
-
|
1145
|
-
Note that `branch` is typically one of:
|
1146
|
-
- `prod`
|
1147
|
-
- `user.bob`
|
1148
|
-
- `test.my_experiment`
|
1149
|
-
- `prod.staging`
|
1150
|
-
|
1151
|
-
|
1152
|
-
Parameters
|
1153
|
-
----------
|
1154
|
-
flow : Union[str, Dict[str, str]], optional, default None
|
1155
|
-
Upstream flow dependency for this flow.
|
1156
|
-
flows : List[Union[str, Dict[str, str]]], default []
|
1157
|
-
Upstream flow dependencies for this flow.
|
1158
|
-
options : Dict[str, Any], default {}
|
1159
|
-
Backend-specific configuration for tuning eventing behavior.
|
1160
|
-
"""
|
1161
|
-
...
|
1162
|
-
|
1163
1027
|
@typing.overload
|
1164
1028
|
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1165
1029
|
"""
|
@@ -1212,94 +1076,95 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
|
|
1212
1076
|
...
|
1213
1077
|
|
1214
1078
|
@typing.overload
|
1215
|
-
def
|
1079
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1216
1080
|
"""
|
1217
|
-
Specifies the
|
1218
|
-
|
1081
|
+
Specifies the event(s) that this flow depends on.
|
1082
|
+
|
1083
|
+
```
|
1084
|
+
@trigger(event='foo')
|
1085
|
+
```
|
1086
|
+
or
|
1087
|
+
```
|
1088
|
+
@trigger(events=['foo', 'bar'])
|
1089
|
+
```
|
1090
|
+
|
1091
|
+
Additionally, you can specify the parameter mappings
|
1092
|
+
to map event payload to Metaflow parameters for the flow.
|
1093
|
+
```
|
1094
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1095
|
+
```
|
1096
|
+
or
|
1097
|
+
```
|
1098
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1099
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1100
|
+
```
|
1101
|
+
|
1102
|
+
'parameters' can also be a list of strings and tuples like so:
|
1103
|
+
```
|
1104
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1105
|
+
```
|
1106
|
+
This is equivalent to:
|
1107
|
+
```
|
1108
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1109
|
+
```
|
1219
1110
|
|
1220
1111
|
|
1221
1112
|
Parameters
|
1222
1113
|
----------
|
1223
|
-
|
1224
|
-
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1228
|
-
|
1229
|
-
cron : str, optional, default None
|
1230
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1231
|
-
specified by this expression.
|
1232
|
-
timezone : str, optional, default None
|
1233
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1234
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1114
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1115
|
+
Event dependency for this flow.
|
1116
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1117
|
+
Events dependency for this flow.
|
1118
|
+
options : Dict[str, Any], default {}
|
1119
|
+
Backend-specific configuration for tuning eventing behavior.
|
1235
1120
|
"""
|
1236
1121
|
...
|
1237
1122
|
|
1238
1123
|
@typing.overload
|
1239
|
-
def
|
1124
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1240
1125
|
...
|
1241
1126
|
|
1242
|
-
def
|
1127
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1243
1128
|
"""
|
1244
|
-
Specifies the
|
1245
|
-
production scheduler.
|
1246
|
-
|
1129
|
+
Specifies the event(s) that this flow depends on.
|
1247
1130
|
|
1248
|
-
|
1249
|
-
|
1250
|
-
|
1251
|
-
|
1252
|
-
|
1253
|
-
|
1254
|
-
|
1255
|
-
Run the workflow weekly.
|
1256
|
-
cron : str, optional, default None
|
1257
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1258
|
-
specified by this expression.
|
1259
|
-
timezone : str, optional, default None
|
1260
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1261
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1262
|
-
"""
|
1263
|
-
...
|
1264
|
-
|
1265
|
-
@typing.overload
|
1266
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1267
|
-
"""
|
1268
|
-
Specifies the PyPI packages for all steps of the flow.
|
1131
|
+
```
|
1132
|
+
@trigger(event='foo')
|
1133
|
+
```
|
1134
|
+
or
|
1135
|
+
```
|
1136
|
+
@trigger(events=['foo', 'bar'])
|
1137
|
+
```
|
1269
1138
|
|
1270
|
-
|
1271
|
-
|
1139
|
+
Additionally, you can specify the parameter mappings
|
1140
|
+
to map event payload to Metaflow parameters for the flow.
|
1141
|
+
```
|
1142
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1143
|
+
```
|
1144
|
+
or
|
1145
|
+
```
|
1146
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1147
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1148
|
+
```
|
1272
1149
|
|
1273
|
-
|
1274
|
-
|
1275
|
-
|
1276
|
-
|
1277
|
-
|
1278
|
-
|
1279
|
-
|
1280
|
-
|
1281
|
-
"""
|
1282
|
-
...
|
1283
|
-
|
1284
|
-
@typing.overload
|
1285
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1286
|
-
...
|
1287
|
-
|
1288
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1289
|
-
"""
|
1290
|
-
Specifies the PyPI packages for all steps of the flow.
|
1150
|
+
'parameters' can also be a list of strings and tuples like so:
|
1151
|
+
```
|
1152
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1153
|
+
```
|
1154
|
+
This is equivalent to:
|
1155
|
+
```
|
1156
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1157
|
+
```
|
1291
1158
|
|
1292
|
-
Use `@pypi_base` to set common packages required by all
|
1293
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1294
1159
|
|
1295
1160
|
Parameters
|
1296
1161
|
----------
|
1297
|
-
|
1298
|
-
|
1299
|
-
|
1300
|
-
|
1301
|
-
|
1302
|
-
|
1162
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1163
|
+
Event dependency for this flow.
|
1164
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1165
|
+
Events dependency for this flow.
|
1166
|
+
options : Dict[str, Any], default {}
|
1167
|
+
Backend-specific configuration for tuning eventing behavior.
|
1303
1168
|
"""
|
1304
1169
|
...
|
1305
1170
|
|
@@ -1346,99 +1211,193 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1346
1211
|
"""
|
1347
1212
|
...
|
1348
1213
|
|
1214
|
+
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1215
|
+
"""
|
1216
|
+
Specifies what flows belong to the same project.
|
1217
|
+
|
1218
|
+
A project-specific namespace is created for all flows that
|
1219
|
+
use the same `@project(name)`.
|
1220
|
+
|
1221
|
+
|
1222
|
+
Parameters
|
1223
|
+
----------
|
1224
|
+
name : str
|
1225
|
+
Project name. Make sure that the name is unique amongst all
|
1226
|
+
projects that use the same production scheduler. The name may
|
1227
|
+
contain only lowercase alphanumeric characters and underscores.
|
1228
|
+
|
1229
|
+
branch : Optional[str], default None
|
1230
|
+
The branch to use. If not specified, the branch is set to
|
1231
|
+
`user.<username>` unless `production` is set to `True`. This can
|
1232
|
+
also be set on the command line using `--branch` as a top-level option.
|
1233
|
+
It is an error to specify `branch` in the decorator and on the command line.
|
1234
|
+
|
1235
|
+
production : bool, default False
|
1236
|
+
Whether or not the branch is the production branch. This can also be set on the
|
1237
|
+
command line using `--production` as a top-level option. It is an error to specify
|
1238
|
+
`production` in the decorator and on the command line.
|
1239
|
+
The project branch name will be:
|
1240
|
+
- if `branch` is specified:
|
1241
|
+
- if `production` is True: `prod.<branch>`
|
1242
|
+
- if `production` is False: `test.<branch>`
|
1243
|
+
- if `branch` is not specified:
|
1244
|
+
- if `production` is True: `prod`
|
1245
|
+
- if `production` is False: `user.<username>`
|
1246
|
+
"""
|
1247
|
+
...
|
1248
|
+
|
1349
1249
|
@typing.overload
|
1350
|
-
def
|
1250
|
+
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1351
1251
|
"""
|
1352
|
-
Specifies the
|
1252
|
+
Specifies the flow(s) that this flow depends on.
|
1353
1253
|
|
1354
1254
|
```
|
1355
|
-
@
|
1255
|
+
@trigger_on_finish(flow='FooFlow')
|
1356
1256
|
```
|
1357
1257
|
or
|
1358
1258
|
```
|
1359
|
-
@
|
1259
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1360
1260
|
```
|
1261
|
+
This decorator respects the @project decorator and triggers the flow
|
1262
|
+
when upstream runs within the same namespace complete successfully
|
1361
1263
|
|
1362
|
-
Additionally, you can specify
|
1363
|
-
|
1264
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1265
|
+
by specifying the fully qualified project_flow_name.
|
1364
1266
|
```
|
1365
|
-
@
|
1267
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1366
1268
|
```
|
1367
1269
|
or
|
1368
1270
|
```
|
1369
|
-
@
|
1370
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1271
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1371
1272
|
```
|
1372
1273
|
|
1373
|
-
|
1374
|
-
|
1375
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1376
|
-
```
|
1377
|
-
This is equivalent to:
|
1274
|
+
You can also specify just the project or project branch (other values will be
|
1275
|
+
inferred from the current project or project branch):
|
1378
1276
|
```
|
1379
|
-
@
|
1277
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1380
1278
|
```
|
1381
1279
|
|
1280
|
+
Note that `branch` is typically one of:
|
1281
|
+
- `prod`
|
1282
|
+
- `user.bob`
|
1283
|
+
- `test.my_experiment`
|
1284
|
+
- `prod.staging`
|
1285
|
+
|
1382
1286
|
|
1383
1287
|
Parameters
|
1384
1288
|
----------
|
1385
|
-
|
1386
|
-
|
1387
|
-
|
1388
|
-
|
1289
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1290
|
+
Upstream flow dependency for this flow.
|
1291
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1292
|
+
Upstream flow dependencies for this flow.
|
1389
1293
|
options : Dict[str, Any], default {}
|
1390
1294
|
Backend-specific configuration for tuning eventing behavior.
|
1391
1295
|
"""
|
1392
1296
|
...
|
1393
1297
|
|
1394
1298
|
@typing.overload
|
1395
|
-
def
|
1299
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1396
1300
|
...
|
1397
1301
|
|
1398
|
-
def
|
1302
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1399
1303
|
"""
|
1400
|
-
Specifies the
|
1304
|
+
Specifies the flow(s) that this flow depends on.
|
1401
1305
|
|
1402
1306
|
```
|
1403
|
-
@
|
1307
|
+
@trigger_on_finish(flow='FooFlow')
|
1404
1308
|
```
|
1405
1309
|
or
|
1406
1310
|
```
|
1407
|
-
@
|
1311
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1408
1312
|
```
|
1313
|
+
This decorator respects the @project decorator and triggers the flow
|
1314
|
+
when upstream runs within the same namespace complete successfully
|
1409
1315
|
|
1410
|
-
Additionally, you can specify
|
1411
|
-
|
1316
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1317
|
+
by specifying the fully qualified project_flow_name.
|
1412
1318
|
```
|
1413
|
-
@
|
1319
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1414
1320
|
```
|
1415
1321
|
or
|
1416
1322
|
```
|
1417
|
-
@
|
1418
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1323
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1419
1324
|
```
|
1420
1325
|
|
1421
|
-
|
1422
|
-
|
1423
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1424
|
-
```
|
1425
|
-
This is equivalent to:
|
1326
|
+
You can also specify just the project or project branch (other values will be
|
1327
|
+
inferred from the current project or project branch):
|
1426
1328
|
```
|
1427
|
-
@
|
1329
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1428
1330
|
```
|
1429
1331
|
|
1332
|
+
Note that `branch` is typically one of:
|
1333
|
+
- `prod`
|
1334
|
+
- `user.bob`
|
1335
|
+
- `test.my_experiment`
|
1336
|
+
- `prod.staging`
|
1337
|
+
|
1430
1338
|
|
1431
1339
|
Parameters
|
1432
1340
|
----------
|
1433
|
-
|
1434
|
-
|
1435
|
-
|
1436
|
-
|
1341
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1342
|
+
Upstream flow dependency for this flow.
|
1343
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1344
|
+
Upstream flow dependencies for this flow.
|
1437
1345
|
options : Dict[str, Any], default {}
|
1438
1346
|
Backend-specific configuration for tuning eventing behavior.
|
1439
1347
|
"""
|
1440
1348
|
...
|
1441
1349
|
|
1350
|
+
@typing.overload
|
1351
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1352
|
+
"""
|
1353
|
+
Specifies the times when the flow should be run when running on a
|
1354
|
+
production scheduler.
|
1355
|
+
|
1356
|
+
|
1357
|
+
Parameters
|
1358
|
+
----------
|
1359
|
+
hourly : bool, default False
|
1360
|
+
Run the workflow hourly.
|
1361
|
+
daily : bool, default True
|
1362
|
+
Run the workflow daily.
|
1363
|
+
weekly : bool, default False
|
1364
|
+
Run the workflow weekly.
|
1365
|
+
cron : str, optional, default None
|
1366
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1367
|
+
specified by this expression.
|
1368
|
+
timezone : str, optional, default None
|
1369
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1370
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1371
|
+
"""
|
1372
|
+
...
|
1373
|
+
|
1374
|
+
@typing.overload
|
1375
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1376
|
+
...
|
1377
|
+
|
1378
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1379
|
+
"""
|
1380
|
+
Specifies the times when the flow should be run when running on a
|
1381
|
+
production scheduler.
|
1382
|
+
|
1383
|
+
|
1384
|
+
Parameters
|
1385
|
+
----------
|
1386
|
+
hourly : bool, default False
|
1387
|
+
Run the workflow hourly.
|
1388
|
+
daily : bool, default True
|
1389
|
+
Run the workflow daily.
|
1390
|
+
weekly : bool, default False
|
1391
|
+
Run the workflow weekly.
|
1392
|
+
cron : str, optional, default None
|
1393
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1394
|
+
specified by this expression.
|
1395
|
+
timezone : str, optional, default None
|
1396
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1397
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1398
|
+
"""
|
1399
|
+
...
|
1400
|
+
|
1442
1401
|
def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
|
1443
1402
|
"""
|
1444
1403
|
Allows setting external datastores to save data for the
|
@@ -1553,5 +1512,46 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
|
|
1553
1512
|
"""
|
1554
1513
|
...
|
1555
1514
|
|
1515
|
+
@typing.overload
|
1516
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1517
|
+
"""
|
1518
|
+
Specifies the PyPI packages for all steps of the flow.
|
1519
|
+
|
1520
|
+
Use `@pypi_base` to set common packages required by all
|
1521
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1522
|
+
|
1523
|
+
Parameters
|
1524
|
+
----------
|
1525
|
+
packages : Dict[str, str], default: {}
|
1526
|
+
Packages to use for this flow. The key is the name of the package
|
1527
|
+
and the value is the version to use.
|
1528
|
+
python : str, optional, default: None
|
1529
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1530
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1531
|
+
"""
|
1532
|
+
...
|
1533
|
+
|
1534
|
+
@typing.overload
|
1535
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1536
|
+
...
|
1537
|
+
|
1538
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1539
|
+
"""
|
1540
|
+
Specifies the PyPI packages for all steps of the flow.
|
1541
|
+
|
1542
|
+
Use `@pypi_base` to set common packages required by all
|
1543
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1544
|
+
|
1545
|
+
Parameters
|
1546
|
+
----------
|
1547
|
+
packages : Dict[str, str], default: {}
|
1548
|
+
Packages to use for this flow. The key is the name of the package
|
1549
|
+
and the value is the version to use.
|
1550
|
+
python : str, optional, default: None
|
1551
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1552
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1553
|
+
"""
|
1554
|
+
...
|
1555
|
+
|
1556
1556
|
pkg_name: str
|
1557
1557
|
|