metaflow-stubs 2.18.8__py2.py3-none-any.whl → 2.18.9__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of metaflow-stubs might be problematic. Click here for more details.
- metaflow-stubs/__init__.pyi +570 -569
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/cli_components/__init__.pyi +2 -2
- metaflow-stubs/cli_components/utils.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +7 -7
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +6 -6
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +13 -6
- metaflow-stubs/meta_files.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +3 -3
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +23 -23
- metaflow-stubs/metaflow_git.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/packaging_sys/__init__.pyi +6 -6
- metaflow-stubs/packaging_sys/backend.pyi +3 -3
- metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
- metaflow-stubs/packaging_sys/tar_backend.pyi +5 -5
- metaflow-stubs/packaging_sys/utils.pyi +2 -2
- metaflow-stubs/packaging_sys/v1.pyi +4 -4
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +13 -12
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +5 -5
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -4
- metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
- metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/parsers.pyi +25 -0
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
- metaflow-stubs/plugins/secrets/utils.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/plugins/uv/__init__.pyi +2 -2
- metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +33 -33
- metaflow-stubs/runner/deployer_impl.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +4 -4
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +4 -4
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/user_configs/__init__.pyi +2 -2
- metaflow-stubs/user_configs/config_options.pyi +3 -3
- metaflow-stubs/user_configs/config_parameters.pyi +5 -5
- metaflow-stubs/user_decorators/__init__.pyi +2 -2
- metaflow-stubs/user_decorators/common.pyi +2 -2
- metaflow-stubs/user_decorators/mutable_flow.pyi +6 -6
- metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
- metaflow-stubs/user_decorators/user_flow_decorator.pyi +4 -4
- metaflow-stubs/user_decorators/user_step_decorator.pyi +7 -7
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.18.8.dist-info → metaflow_stubs-2.18.9.dist-info}/METADATA +2 -2
- metaflow_stubs-2.18.9.dist-info/RECORD +167 -0
- metaflow_stubs-2.18.8.dist-info/RECORD +0 -166
- {metaflow_stubs-2.18.8.dist-info → metaflow_stubs-2.18.9.dist-info}/WHEEL +0 -0
- {metaflow_stubs-2.18.8.dist-info → metaflow_stubs-2.18.9.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
######################################################################################################
|
|
2
2
|
# Auto-generated Metaflow stub file #
|
|
3
|
-
# MF version: 2.18.
|
|
4
|
-
# Generated on 2025-09-
|
|
3
|
+
# MF version: 2.18.9 #
|
|
4
|
+
# Generated on 2025-09-24T20:09:24.398546 #
|
|
5
5
|
######################################################################################################
|
|
6
6
|
|
|
7
7
|
from __future__ import annotations
|
|
@@ -40,16 +40,17 @@ from .user_decorators.user_step_decorator import StepMutator as StepMutator
|
|
|
40
40
|
from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
|
|
41
41
|
from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
|
|
42
42
|
from . import tuple_util as tuple_util
|
|
43
|
-
from . import metaflow_git as metaflow_git
|
|
44
43
|
from . import events as events
|
|
44
|
+
from . import metaflow_git as metaflow_git
|
|
45
45
|
from . import runner as runner
|
|
46
46
|
from . import plugins as plugins
|
|
47
47
|
from .plugins.datatools.s3.s3 import S3 as S3
|
|
48
48
|
from . import includefile as includefile
|
|
49
49
|
from .includefile import IncludeFile as IncludeFile
|
|
50
|
+
from .plugins.parsers import yaml_parser as yaml_parser
|
|
51
|
+
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
|
50
52
|
from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
|
|
51
53
|
from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
|
|
52
|
-
from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
|
|
53
54
|
from . import cards as cards
|
|
54
55
|
from . import client as client
|
|
55
56
|
from .client.core import namespace as namespace
|
|
@@ -152,119 +153,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
|
152
153
|
"""
|
|
153
154
|
...
|
|
154
155
|
|
|
155
|
-
@typing.overload
|
|
156
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
157
|
-
"""
|
|
158
|
-
Specifies the number of times the task corresponding
|
|
159
|
-
to a step needs to be retried.
|
|
160
|
-
|
|
161
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
|
162
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
163
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
|
164
|
-
|
|
165
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
166
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
|
167
|
-
ensuring that the flow execution can continue.
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
Parameters
|
|
171
|
-
----------
|
|
172
|
-
times : int, default 3
|
|
173
|
-
Number of times to retry this task.
|
|
174
|
-
minutes_between_retries : int, default 2
|
|
175
|
-
Number of minutes between retries.
|
|
176
|
-
"""
|
|
177
|
-
...
|
|
178
|
-
|
|
179
|
-
@typing.overload
|
|
180
|
-
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
181
|
-
...
|
|
182
|
-
|
|
183
|
-
@typing.overload
|
|
184
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
185
|
-
...
|
|
186
|
-
|
|
187
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
|
188
|
-
"""
|
|
189
|
-
Specifies the number of times the task corresponding
|
|
190
|
-
to a step needs to be retried.
|
|
191
|
-
|
|
192
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
|
193
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
194
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
|
195
|
-
|
|
196
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
197
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
|
198
|
-
ensuring that the flow execution can continue.
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
Parameters
|
|
202
|
-
----------
|
|
203
|
-
times : int, default 3
|
|
204
|
-
Number of times to retry this task.
|
|
205
|
-
minutes_between_retries : int, default 2
|
|
206
|
-
Number of minutes between retries.
|
|
207
|
-
"""
|
|
208
|
-
...
|
|
209
|
-
|
|
210
|
-
@typing.overload
|
|
211
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
212
|
-
"""
|
|
213
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
214
|
-
the execution of a step.
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
Parameters
|
|
218
|
-
----------
|
|
219
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
220
|
-
List of secret specs, defining how the secrets are to be retrieved
|
|
221
|
-
role : str, optional, default: None
|
|
222
|
-
Role to use for fetching secrets
|
|
223
|
-
"""
|
|
224
|
-
...
|
|
225
|
-
|
|
226
|
-
@typing.overload
|
|
227
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
228
|
-
...
|
|
229
|
-
|
|
230
|
-
@typing.overload
|
|
231
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
232
|
-
...
|
|
233
|
-
|
|
234
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
|
235
|
-
"""
|
|
236
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
237
|
-
the execution of a step.
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
Parameters
|
|
241
|
-
----------
|
|
242
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
243
|
-
List of secret specs, defining how the secrets are to be retrieved
|
|
244
|
-
role : str, optional, default: None
|
|
245
|
-
Role to use for fetching secrets
|
|
246
|
-
"""
|
|
247
|
-
...
|
|
248
|
-
|
|
249
|
-
@typing.overload
|
|
250
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
251
|
-
"""
|
|
252
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
253
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
254
|
-
"""
|
|
255
|
-
...
|
|
256
|
-
|
|
257
|
-
@typing.overload
|
|
258
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
259
|
-
...
|
|
260
|
-
|
|
261
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
262
|
-
"""
|
|
263
|
-
Decorator prototype for all step decorators. This function gets specialized
|
|
264
|
-
and imported for all decorators types by _import_plugin_decorators().
|
|
265
|
-
"""
|
|
266
|
-
...
|
|
267
|
-
|
|
268
156
|
@typing.overload
|
|
269
157
|
def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, aws_batch_tags: typing.Optional[typing.Dict[str, str]] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
270
158
|
"""
|
|
@@ -420,98 +308,6 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
|
420
308
|
"""
|
|
421
309
|
...
|
|
422
310
|
|
|
423
|
-
@typing.overload
|
|
424
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
425
|
-
"""
|
|
426
|
-
Specifies the Conda environment for the step.
|
|
427
|
-
|
|
428
|
-
Information in this decorator will augment any
|
|
429
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
430
|
-
you can use `@conda_base` to set packages required by all
|
|
431
|
-
steps and use `@conda` to specify step-specific overrides.
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
Parameters
|
|
435
|
-
----------
|
|
436
|
-
packages : Dict[str, str], default {}
|
|
437
|
-
Packages to use for this step. The key is the name of the package
|
|
438
|
-
and the value is the version to use.
|
|
439
|
-
libraries : Dict[str, str], default {}
|
|
440
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
441
|
-
python : str, optional, default None
|
|
442
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
443
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
444
|
-
disabled : bool, default False
|
|
445
|
-
If set to True, disables @conda.
|
|
446
|
-
"""
|
|
447
|
-
...
|
|
448
|
-
|
|
449
|
-
@typing.overload
|
|
450
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
451
|
-
...
|
|
452
|
-
|
|
453
|
-
@typing.overload
|
|
454
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
455
|
-
...
|
|
456
|
-
|
|
457
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
458
|
-
"""
|
|
459
|
-
Specifies the Conda environment for the step.
|
|
460
|
-
|
|
461
|
-
Information in this decorator will augment any
|
|
462
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
463
|
-
you can use `@conda_base` to set packages required by all
|
|
464
|
-
steps and use `@conda` to specify step-specific overrides.
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
Parameters
|
|
468
|
-
----------
|
|
469
|
-
packages : Dict[str, str], default {}
|
|
470
|
-
Packages to use for this step. The key is the name of the package
|
|
471
|
-
and the value is the version to use.
|
|
472
|
-
libraries : Dict[str, str], default {}
|
|
473
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
474
|
-
python : str, optional, default None
|
|
475
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
476
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
477
|
-
disabled : bool, default False
|
|
478
|
-
If set to True, disables @conda.
|
|
479
|
-
"""
|
|
480
|
-
...
|
|
481
|
-
|
|
482
|
-
@typing.overload
|
|
483
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
484
|
-
"""
|
|
485
|
-
Specifies environment variables to be set prior to the execution of a step.
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
Parameters
|
|
489
|
-
----------
|
|
490
|
-
vars : Dict[str, str], default {}
|
|
491
|
-
Dictionary of environment variables to set.
|
|
492
|
-
"""
|
|
493
|
-
...
|
|
494
|
-
|
|
495
|
-
@typing.overload
|
|
496
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
497
|
-
...
|
|
498
|
-
|
|
499
|
-
@typing.overload
|
|
500
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
501
|
-
...
|
|
502
|
-
|
|
503
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
|
504
|
-
"""
|
|
505
|
-
Specifies environment variables to be set prior to the execution of a step.
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
Parameters
|
|
509
|
-
----------
|
|
510
|
-
vars : Dict[str, str], default {}
|
|
511
|
-
Dictionary of environment variables to set.
|
|
512
|
-
"""
|
|
513
|
-
...
|
|
514
|
-
|
|
515
311
|
@typing.overload
|
|
516
312
|
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
517
313
|
"""
|
|
@@ -572,62 +368,39 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
|
572
368
|
...
|
|
573
369
|
|
|
574
370
|
@typing.overload
|
|
575
|
-
def
|
|
371
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
576
372
|
"""
|
|
577
|
-
|
|
373
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
578
374
|
|
|
579
|
-
|
|
580
|
-
contains the exception raised. You can use it to detect the presence
|
|
581
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
|
582
|
-
are missing.
|
|
375
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
583
376
|
|
|
584
377
|
|
|
585
378
|
Parameters
|
|
586
379
|
----------
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
380
|
+
type : str, default 'default'
|
|
381
|
+
Card type.
|
|
382
|
+
id : str, optional, default None
|
|
383
|
+
If multiple cards are present, use this id to identify this card.
|
|
384
|
+
options : Dict[str, Any], default {}
|
|
385
|
+
Options passed to the card. The contents depend on the card type.
|
|
386
|
+
timeout : int, default 45
|
|
387
|
+
Interrupt reporting if it takes more than this many seconds.
|
|
593
388
|
"""
|
|
594
389
|
...
|
|
595
390
|
|
|
596
391
|
@typing.overload
|
|
597
|
-
def
|
|
392
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
598
393
|
...
|
|
599
394
|
|
|
600
395
|
@typing.overload
|
|
601
|
-
def
|
|
396
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
602
397
|
...
|
|
603
398
|
|
|
604
|
-
def
|
|
399
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
|
605
400
|
"""
|
|
606
|
-
|
|
401
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
607
402
|
|
|
608
|
-
|
|
609
|
-
contains the exception raised. You can use it to detect the presence
|
|
610
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
|
611
|
-
are missing.
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
Parameters
|
|
615
|
-
----------
|
|
616
|
-
var : str, optional, default None
|
|
617
|
-
Name of the artifact in which to store the caught exception.
|
|
618
|
-
If not specified, the exception is not stored.
|
|
619
|
-
print_exception : bool, default True
|
|
620
|
-
Determines whether or not the exception is printed to
|
|
621
|
-
stdout when caught.
|
|
622
|
-
"""
|
|
623
|
-
...
|
|
624
|
-
|
|
625
|
-
@typing.overload
|
|
626
|
-
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
627
|
-
"""
|
|
628
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
|
629
|
-
|
|
630
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
403
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
|
631
404
|
|
|
632
405
|
|
|
633
406
|
Parameters
|
|
@@ -644,30 +417,309 @@ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typ
|
|
|
644
417
|
...
|
|
645
418
|
|
|
646
419
|
@typing.overload
|
|
647
|
-
def
|
|
420
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
421
|
+
"""
|
|
422
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
423
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
424
|
+
"""
|
|
648
425
|
...
|
|
649
426
|
|
|
650
427
|
@typing.overload
|
|
651
|
-
def
|
|
428
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
652
429
|
...
|
|
653
430
|
|
|
654
|
-
def
|
|
431
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
|
655
432
|
"""
|
|
656
|
-
|
|
433
|
+
Decorator prototype for all step decorators. This function gets specialized
|
|
434
|
+
and imported for all decorators types by _import_plugin_decorators().
|
|
435
|
+
"""
|
|
436
|
+
...
|
|
437
|
+
|
|
438
|
+
@typing.overload
|
|
439
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
440
|
+
"""
|
|
441
|
+
Specifies that the step will success under all circumstances.
|
|
657
442
|
|
|
658
|
-
|
|
443
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
444
|
+
contains the exception raised. You can use it to detect the presence
|
|
445
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
446
|
+
are missing.
|
|
659
447
|
|
|
660
448
|
|
|
661
449
|
Parameters
|
|
662
450
|
----------
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
451
|
+
var : str, optional, default None
|
|
452
|
+
Name of the artifact in which to store the caught exception.
|
|
453
|
+
If not specified, the exception is not stored.
|
|
454
|
+
print_exception : bool, default True
|
|
455
|
+
Determines whether or not the exception is printed to
|
|
456
|
+
stdout when caught.
|
|
457
|
+
"""
|
|
458
|
+
...
|
|
459
|
+
|
|
460
|
+
@typing.overload
|
|
461
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
462
|
+
...
|
|
463
|
+
|
|
464
|
+
@typing.overload
|
|
465
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
466
|
+
...
|
|
467
|
+
|
|
468
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
|
469
|
+
"""
|
|
470
|
+
Specifies that the step will success under all circumstances.
|
|
471
|
+
|
|
472
|
+
The decorator will create an optional artifact, specified by `var`, which
|
|
473
|
+
contains the exception raised. You can use it to detect the presence
|
|
474
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
|
475
|
+
are missing.
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
Parameters
|
|
479
|
+
----------
|
|
480
|
+
var : str, optional, default None
|
|
481
|
+
Name of the artifact in which to store the caught exception.
|
|
482
|
+
If not specified, the exception is not stored.
|
|
483
|
+
print_exception : bool, default True
|
|
484
|
+
Determines whether or not the exception is printed to
|
|
485
|
+
stdout when caught.
|
|
486
|
+
"""
|
|
487
|
+
...
|
|
488
|
+
|
|
489
|
+
@typing.overload
|
|
490
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
491
|
+
"""
|
|
492
|
+
Specifies the number of times the task corresponding
|
|
493
|
+
to a step needs to be retried.
|
|
494
|
+
|
|
495
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
|
496
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
497
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
|
498
|
+
|
|
499
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
500
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
|
501
|
+
ensuring that the flow execution can continue.
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
Parameters
|
|
505
|
+
----------
|
|
506
|
+
times : int, default 3
|
|
507
|
+
Number of times to retry this task.
|
|
508
|
+
minutes_between_retries : int, default 2
|
|
509
|
+
Number of minutes between retries.
|
|
510
|
+
"""
|
|
511
|
+
...
|
|
512
|
+
|
|
513
|
+
@typing.overload
|
|
514
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
515
|
+
...
|
|
516
|
+
|
|
517
|
+
@typing.overload
|
|
518
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
519
|
+
...
|
|
520
|
+
|
|
521
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
|
522
|
+
"""
|
|
523
|
+
Specifies the number of times the task corresponding
|
|
524
|
+
to a step needs to be retried.
|
|
525
|
+
|
|
526
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
|
527
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
|
528
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
|
529
|
+
|
|
530
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
|
531
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
|
532
|
+
ensuring that the flow execution can continue.
|
|
533
|
+
|
|
534
|
+
|
|
535
|
+
Parameters
|
|
536
|
+
----------
|
|
537
|
+
times : int, default 3
|
|
538
|
+
Number of times to retry this task.
|
|
539
|
+
minutes_between_retries : int, default 2
|
|
540
|
+
Number of minutes between retries.
|
|
541
|
+
"""
|
|
542
|
+
...
|
|
543
|
+
|
|
544
|
+
@typing.overload
|
|
545
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
546
|
+
"""
|
|
547
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
Parameters
|
|
551
|
+
----------
|
|
552
|
+
vars : Dict[str, str], default {}
|
|
553
|
+
Dictionary of environment variables to set.
|
|
554
|
+
"""
|
|
555
|
+
...
|
|
556
|
+
|
|
557
|
+
@typing.overload
|
|
558
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
559
|
+
...
|
|
560
|
+
|
|
561
|
+
@typing.overload
|
|
562
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
563
|
+
...
|
|
564
|
+
|
|
565
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
|
566
|
+
"""
|
|
567
|
+
Specifies environment variables to be set prior to the execution of a step.
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
Parameters
|
|
571
|
+
----------
|
|
572
|
+
vars : Dict[str, str], default {}
|
|
573
|
+
Dictionary of environment variables to set.
|
|
574
|
+
"""
|
|
575
|
+
...
|
|
576
|
+
|
|
577
|
+
@typing.overload
|
|
578
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
579
|
+
"""
|
|
580
|
+
Specifies the PyPI packages for the step.
|
|
581
|
+
|
|
582
|
+
Information in this decorator will augment any
|
|
583
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
584
|
+
you can use `@pypi_base` to set packages required by all
|
|
585
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
586
|
+
|
|
587
|
+
|
|
588
|
+
Parameters
|
|
589
|
+
----------
|
|
590
|
+
packages : Dict[str, str], default: {}
|
|
591
|
+
Packages to use for this step. The key is the name of the package
|
|
592
|
+
and the value is the version to use.
|
|
593
|
+
python : str, optional, default: None
|
|
594
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
595
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
596
|
+
"""
|
|
597
|
+
...
|
|
598
|
+
|
|
599
|
+
@typing.overload
|
|
600
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
601
|
+
...
|
|
602
|
+
|
|
603
|
+
@typing.overload
|
|
604
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
605
|
+
...
|
|
606
|
+
|
|
607
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
608
|
+
"""
|
|
609
|
+
Specifies the PyPI packages for the step.
|
|
610
|
+
|
|
611
|
+
Information in this decorator will augment any
|
|
612
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
613
|
+
you can use `@pypi_base` to set packages required by all
|
|
614
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
Parameters
|
|
618
|
+
----------
|
|
619
|
+
packages : Dict[str, str], default: {}
|
|
620
|
+
Packages to use for this step. The key is the name of the package
|
|
621
|
+
and the value is the version to use.
|
|
622
|
+
python : str, optional, default: None
|
|
623
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
624
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
625
|
+
"""
|
|
626
|
+
...
|
|
627
|
+
|
|
628
|
+
@typing.overload
|
|
629
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
630
|
+
"""
|
|
631
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
632
|
+
the execution of a step.
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
Parameters
|
|
636
|
+
----------
|
|
637
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
638
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
639
|
+
role : str, optional, default: None
|
|
640
|
+
Role to use for fetching secrets
|
|
641
|
+
"""
|
|
642
|
+
...
|
|
643
|
+
|
|
644
|
+
@typing.overload
|
|
645
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
646
|
+
...
|
|
647
|
+
|
|
648
|
+
@typing.overload
|
|
649
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
650
|
+
...
|
|
651
|
+
|
|
652
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
|
|
653
|
+
"""
|
|
654
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
|
655
|
+
the execution of a step.
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
Parameters
|
|
659
|
+
----------
|
|
660
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
|
661
|
+
List of secret specs, defining how the secrets are to be retrieved
|
|
662
|
+
role : str, optional, default: None
|
|
663
|
+
Role to use for fetching secrets
|
|
664
|
+
"""
|
|
665
|
+
...
|
|
666
|
+
|
|
667
|
+
@typing.overload
|
|
668
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
669
|
+
"""
|
|
670
|
+
Specifies the Conda environment for the step.
|
|
671
|
+
|
|
672
|
+
Information in this decorator will augment any
|
|
673
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
674
|
+
you can use `@conda_base` to set packages required by all
|
|
675
|
+
steps and use `@conda` to specify step-specific overrides.
|
|
676
|
+
|
|
677
|
+
|
|
678
|
+
Parameters
|
|
679
|
+
----------
|
|
680
|
+
packages : Dict[str, str], default {}
|
|
681
|
+
Packages to use for this step. The key is the name of the package
|
|
682
|
+
and the value is the version to use.
|
|
683
|
+
libraries : Dict[str, str], default {}
|
|
684
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
685
|
+
python : str, optional, default None
|
|
686
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
687
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
688
|
+
disabled : bool, default False
|
|
689
|
+
If set to True, disables @conda.
|
|
690
|
+
"""
|
|
691
|
+
...
|
|
692
|
+
|
|
693
|
+
@typing.overload
|
|
694
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
695
|
+
...
|
|
696
|
+
|
|
697
|
+
@typing.overload
|
|
698
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
699
|
+
...
|
|
700
|
+
|
|
701
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
702
|
+
"""
|
|
703
|
+
Specifies the Conda environment for the step.
|
|
704
|
+
|
|
705
|
+
Information in this decorator will augment any
|
|
706
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
|
707
|
+
you can use `@conda_base` to set packages required by all
|
|
708
|
+
steps and use `@conda` to specify step-specific overrides.
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
Parameters
|
|
712
|
+
----------
|
|
713
|
+
packages : Dict[str, str], default {}
|
|
714
|
+
Packages to use for this step. The key is the name of the package
|
|
715
|
+
and the value is the version to use.
|
|
716
|
+
libraries : Dict[str, str], default {}
|
|
717
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
718
|
+
python : str, optional, default None
|
|
719
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
720
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
721
|
+
disabled : bool, default False
|
|
722
|
+
If set to True, disables @conda.
|
|
671
723
|
"""
|
|
672
724
|
...
|
|
673
725
|
|
|
@@ -746,68 +798,17 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
|
746
798
|
If not specified, any accessible compute pool within the perimeter is used.
|
|
747
799
|
hostname_resolution_timeout: int, default 10 * 60
|
|
748
800
|
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
|
749
|
-
Only applicable when @parallel is used.
|
|
750
|
-
qos: str, default: Burstable
|
|
751
|
-
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
|
752
|
-
|
|
753
|
-
security_context: Dict[str, Any], optional, default None
|
|
754
|
-
Container security context. Applies to the task container. Allows the following keys:
|
|
755
|
-
- privileged: bool, optional, default None
|
|
756
|
-
- allow_privilege_escalation: bool, optional, default None
|
|
757
|
-
- run_as_user: int, optional, default None
|
|
758
|
-
- run_as_group: int, optional, default None
|
|
759
|
-
- run_as_non_root: bool, optional, default None
|
|
760
|
-
"""
|
|
761
|
-
...
|
|
762
|
-
|
|
763
|
-
@typing.overload
|
|
764
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
|
765
|
-
"""
|
|
766
|
-
Specifies the PyPI packages for the step.
|
|
767
|
-
|
|
768
|
-
Information in this decorator will augment any
|
|
769
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
770
|
-
you can use `@pypi_base` to set packages required by all
|
|
771
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
Parameters
|
|
775
|
-
----------
|
|
776
|
-
packages : Dict[str, str], default: {}
|
|
777
|
-
Packages to use for this step. The key is the name of the package
|
|
778
|
-
and the value is the version to use.
|
|
779
|
-
python : str, optional, default: None
|
|
780
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
781
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
782
|
-
"""
|
|
783
|
-
...
|
|
784
|
-
|
|
785
|
-
@typing.overload
|
|
786
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
|
787
|
-
...
|
|
788
|
-
|
|
789
|
-
@typing.overload
|
|
790
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
|
791
|
-
...
|
|
792
|
-
|
|
793
|
-
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
794
|
-
"""
|
|
795
|
-
Specifies the PyPI packages for the step.
|
|
796
|
-
|
|
797
|
-
Information in this decorator will augment any
|
|
798
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
|
799
|
-
you can use `@pypi_base` to set packages required by all
|
|
800
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
801
|
-
|
|
801
|
+
Only applicable when @parallel is used.
|
|
802
|
+
qos: str, default: Burstable
|
|
803
|
+
Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
|
|
802
804
|
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
805
|
+
security_context: Dict[str, Any], optional, default None
|
|
806
|
+
Container security context. Applies to the task container. Allows the following keys:
|
|
807
|
+
- privileged: bool, optional, default None
|
|
808
|
+
- allow_privilege_escalation: bool, optional, default None
|
|
809
|
+
- run_as_user: int, optional, default None
|
|
810
|
+
- run_as_group: int, optional, default None
|
|
811
|
+
- run_as_non_root: bool, optional, default None
|
|
811
812
|
"""
|
|
812
813
|
...
|
|
813
814
|
|
|
@@ -890,6 +891,148 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
|
890
891
|
"""
|
|
891
892
|
...
|
|
892
893
|
|
|
894
|
+
@typing.overload
|
|
895
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
896
|
+
"""
|
|
897
|
+
Specifies the PyPI packages for all steps of the flow.
|
|
898
|
+
|
|
899
|
+
Use `@pypi_base` to set common packages required by all
|
|
900
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
901
|
+
|
|
902
|
+
Parameters
|
|
903
|
+
----------
|
|
904
|
+
packages : Dict[str, str], default: {}
|
|
905
|
+
Packages to use for this flow. The key is the name of the package
|
|
906
|
+
and the value is the version to use.
|
|
907
|
+
python : str, optional, default: None
|
|
908
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
909
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
910
|
+
"""
|
|
911
|
+
...
|
|
912
|
+
|
|
913
|
+
@typing.overload
|
|
914
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
915
|
+
...
|
|
916
|
+
|
|
917
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
918
|
+
"""
|
|
919
|
+
Specifies the PyPI packages for all steps of the flow.
|
|
920
|
+
|
|
921
|
+
Use `@pypi_base` to set common packages required by all
|
|
922
|
+
steps and use `@pypi` to specify step-specific overrides.
|
|
923
|
+
|
|
924
|
+
Parameters
|
|
925
|
+
----------
|
|
926
|
+
packages : Dict[str, str], default: {}
|
|
927
|
+
Packages to use for this flow. The key is the name of the package
|
|
928
|
+
and the value is the version to use.
|
|
929
|
+
python : str, optional, default: None
|
|
930
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
931
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
932
|
+
"""
|
|
933
|
+
...
|
|
934
|
+
|
|
935
|
+
@typing.overload
|
|
936
|
+
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
937
|
+
"""
|
|
938
|
+
Specifies the flow(s) that this flow depends on.
|
|
939
|
+
|
|
940
|
+
```
|
|
941
|
+
@trigger_on_finish(flow='FooFlow')
|
|
942
|
+
```
|
|
943
|
+
or
|
|
944
|
+
```
|
|
945
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
946
|
+
```
|
|
947
|
+
This decorator respects the @project decorator and triggers the flow
|
|
948
|
+
when upstream runs within the same namespace complete successfully
|
|
949
|
+
|
|
950
|
+
Additionally, you can specify project aware upstream flow dependencies
|
|
951
|
+
by specifying the fully qualified project_flow_name.
|
|
952
|
+
```
|
|
953
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
|
954
|
+
```
|
|
955
|
+
or
|
|
956
|
+
```
|
|
957
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
|
958
|
+
```
|
|
959
|
+
|
|
960
|
+
You can also specify just the project or project branch (other values will be
|
|
961
|
+
inferred from the current project or project branch):
|
|
962
|
+
```
|
|
963
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
|
964
|
+
```
|
|
965
|
+
|
|
966
|
+
Note that `branch` is typically one of:
|
|
967
|
+
- `prod`
|
|
968
|
+
- `user.bob`
|
|
969
|
+
- `test.my_experiment`
|
|
970
|
+
- `prod.staging`
|
|
971
|
+
|
|
972
|
+
|
|
973
|
+
Parameters
|
|
974
|
+
----------
|
|
975
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
|
976
|
+
Upstream flow dependency for this flow.
|
|
977
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
|
978
|
+
Upstream flow dependencies for this flow.
|
|
979
|
+
options : Dict[str, Any], default {}
|
|
980
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
981
|
+
"""
|
|
982
|
+
...
|
|
983
|
+
|
|
984
|
+
@typing.overload
|
|
985
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
986
|
+
...
|
|
987
|
+
|
|
988
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
|
989
|
+
"""
|
|
990
|
+
Specifies the flow(s) that this flow depends on.
|
|
991
|
+
|
|
992
|
+
```
|
|
993
|
+
@trigger_on_finish(flow='FooFlow')
|
|
994
|
+
```
|
|
995
|
+
or
|
|
996
|
+
```
|
|
997
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
998
|
+
```
|
|
999
|
+
This decorator respects the @project decorator and triggers the flow
|
|
1000
|
+
when upstream runs within the same namespace complete successfully
|
|
1001
|
+
|
|
1002
|
+
Additionally, you can specify project aware upstream flow dependencies
|
|
1003
|
+
by specifying the fully qualified project_flow_name.
|
|
1004
|
+
```
|
|
1005
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
|
1006
|
+
```
|
|
1007
|
+
or
|
|
1008
|
+
```
|
|
1009
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
|
1010
|
+
```
|
|
1011
|
+
|
|
1012
|
+
You can also specify just the project or project branch (other values will be
|
|
1013
|
+
inferred from the current project or project branch):
|
|
1014
|
+
```
|
|
1015
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
|
1016
|
+
```
|
|
1017
|
+
|
|
1018
|
+
Note that `branch` is typically one of:
|
|
1019
|
+
- `prod`
|
|
1020
|
+
- `user.bob`
|
|
1021
|
+
- `test.my_experiment`
|
|
1022
|
+
- `prod.staging`
|
|
1023
|
+
|
|
1024
|
+
|
|
1025
|
+
Parameters
|
|
1026
|
+
----------
|
|
1027
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
|
1028
|
+
Upstream flow dependency for this flow.
|
|
1029
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
|
1030
|
+
Upstream flow dependencies for this flow.
|
|
1031
|
+
options : Dict[str, Any], default {}
|
|
1032
|
+
Backend-specific configuration for tuning eventing behavior.
|
|
1033
|
+
"""
|
|
1034
|
+
...
|
|
1035
|
+
|
|
893
1036
|
@typing.overload
|
|
894
1037
|
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
895
1038
|
"""
|
|
@@ -941,104 +1084,132 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
|
|
|
941
1084
|
"""
|
|
942
1085
|
...
|
|
943
1086
|
|
|
944
|
-
|
|
945
|
-
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1087
|
+
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
946
1088
|
"""
|
|
947
|
-
Specifies
|
|
1089
|
+
Specifies what flows belong to the same project.
|
|
948
1090
|
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
```
|
|
952
|
-
or
|
|
953
|
-
```
|
|
954
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
955
|
-
```
|
|
956
|
-
This decorator respects the @project decorator and triggers the flow
|
|
957
|
-
when upstream runs within the same namespace complete successfully
|
|
1091
|
+
A project-specific namespace is created for all flows that
|
|
1092
|
+
use the same `@project(name)`.
|
|
958
1093
|
|
|
959
|
-
Additionally, you can specify project aware upstream flow dependencies
|
|
960
|
-
by specifying the fully qualified project_flow_name.
|
|
961
|
-
```
|
|
962
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
|
963
|
-
```
|
|
964
|
-
or
|
|
965
|
-
```
|
|
966
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
|
967
|
-
```
|
|
968
1094
|
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
1095
|
+
Parameters
|
|
1096
|
+
----------
|
|
1097
|
+
name : str
|
|
1098
|
+
Project name. Make sure that the name is unique amongst all
|
|
1099
|
+
projects that use the same production scheduler. The name may
|
|
1100
|
+
contain only lowercase alphanumeric characters and underscores.
|
|
1101
|
+
|
|
1102
|
+
branch : Optional[str], default None
|
|
1103
|
+
The branch to use. If not specified, the branch is set to
|
|
1104
|
+
`user.<username>` unless `production` is set to `True`. This can
|
|
1105
|
+
also be set on the command line using `--branch` as a top-level option.
|
|
1106
|
+
It is an error to specify `branch` in the decorator and on the command line.
|
|
1107
|
+
|
|
1108
|
+
production : bool, default False
|
|
1109
|
+
Whether or not the branch is the production branch. This can also be set on the
|
|
1110
|
+
command line using `--production` as a top-level option. It is an error to specify
|
|
1111
|
+
`production` in the decorator and on the command line.
|
|
1112
|
+
The project branch name will be:
|
|
1113
|
+
- if `branch` is specified:
|
|
1114
|
+
- if `production` is True: `prod.<branch>`
|
|
1115
|
+
- if `production` is False: `test.<branch>`
|
|
1116
|
+
- if `branch` is not specified:
|
|
1117
|
+
- if `production` is True: `prod`
|
|
1118
|
+
- if `production` is False: `user.<username>`
|
|
1119
|
+
"""
|
|
1120
|
+
...
|
|
1121
|
+
|
|
1122
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1123
|
+
"""
|
|
1124
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1125
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1126
|
+
|
|
1127
|
+
|
|
1128
|
+
Parameters
|
|
1129
|
+
----------
|
|
1130
|
+
timeout : int
|
|
1131
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1132
|
+
poke_interval : int
|
|
1133
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1134
|
+
mode : str
|
|
1135
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1136
|
+
exponential_backoff : bool
|
|
1137
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1138
|
+
pool : str
|
|
1139
|
+
the slot pool this task should run in,
|
|
1140
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1141
|
+
soft_fail : bool
|
|
1142
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1143
|
+
name : str
|
|
1144
|
+
Name of the sensor on Airflow
|
|
1145
|
+
description : str
|
|
1146
|
+
Description of sensor in the Airflow UI
|
|
1147
|
+
external_dag_id : str
|
|
1148
|
+
The dag_id that contains the task you want to wait for.
|
|
1149
|
+
external_task_ids : List[str]
|
|
1150
|
+
The list of task_ids that you want to wait for.
|
|
1151
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1152
|
+
allowed_states : List[str]
|
|
1153
|
+
Iterable of allowed states, (Default: ['success'])
|
|
1154
|
+
failed_states : List[str]
|
|
1155
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
|
1156
|
+
execution_delta : datetime.timedelta
|
|
1157
|
+
time difference with the previous execution to look at,
|
|
1158
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1159
|
+
check_existence: bool
|
|
1160
|
+
Set to True to check if the external task exists or check if
|
|
1161
|
+
the DAG to wait for exists. (Default: True)
|
|
1162
|
+
"""
|
|
1163
|
+
...
|
|
1164
|
+
|
|
1165
|
+
@typing.overload
|
|
1166
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1167
|
+
"""
|
|
1168
|
+
Specifies the Conda environment for all steps of the flow.
|
|
974
1169
|
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
- `user.bob`
|
|
978
|
-
- `test.my_experiment`
|
|
979
|
-
- `prod.staging`
|
|
1170
|
+
Use `@conda_base` to set common libraries required by all
|
|
1171
|
+
steps and use `@conda` to specify step-specific additions.
|
|
980
1172
|
|
|
981
1173
|
|
|
982
1174
|
Parameters
|
|
983
1175
|
----------
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
1176
|
+
packages : Dict[str, str], default {}
|
|
1177
|
+
Packages to use for this flow. The key is the name of the package
|
|
1178
|
+
and the value is the version to use.
|
|
1179
|
+
libraries : Dict[str, str], default {}
|
|
1180
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1181
|
+
python : str, optional, default None
|
|
1182
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1183
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1184
|
+
disabled : bool, default False
|
|
1185
|
+
If set to True, disables Conda.
|
|
990
1186
|
"""
|
|
991
1187
|
...
|
|
992
1188
|
|
|
993
1189
|
@typing.overload
|
|
994
|
-
def
|
|
1190
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
995
1191
|
...
|
|
996
1192
|
|
|
997
|
-
def
|
|
1193
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
998
1194
|
"""
|
|
999
|
-
Specifies the
|
|
1000
|
-
|
|
1001
|
-
```
|
|
1002
|
-
@trigger_on_finish(flow='FooFlow')
|
|
1003
|
-
```
|
|
1004
|
-
or
|
|
1005
|
-
```
|
|
1006
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
|
1007
|
-
```
|
|
1008
|
-
This decorator respects the @project decorator and triggers the flow
|
|
1009
|
-
when upstream runs within the same namespace complete successfully
|
|
1010
|
-
|
|
1011
|
-
Additionally, you can specify project aware upstream flow dependencies
|
|
1012
|
-
by specifying the fully qualified project_flow_name.
|
|
1013
|
-
```
|
|
1014
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
|
1015
|
-
```
|
|
1016
|
-
or
|
|
1017
|
-
```
|
|
1018
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
|
1019
|
-
```
|
|
1020
|
-
|
|
1021
|
-
You can also specify just the project or project branch (other values will be
|
|
1022
|
-
inferred from the current project or project branch):
|
|
1023
|
-
```
|
|
1024
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
|
1025
|
-
```
|
|
1195
|
+
Specifies the Conda environment for all steps of the flow.
|
|
1026
1196
|
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
- `user.bob`
|
|
1030
|
-
- `test.my_experiment`
|
|
1031
|
-
- `prod.staging`
|
|
1197
|
+
Use `@conda_base` to set common libraries required by all
|
|
1198
|
+
steps and use `@conda` to specify step-specific additions.
|
|
1032
1199
|
|
|
1033
1200
|
|
|
1034
1201
|
Parameters
|
|
1035
1202
|
----------
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1203
|
+
packages : Dict[str, str], default {}
|
|
1204
|
+
Packages to use for this flow. The key is the name of the package
|
|
1205
|
+
and the value is the version to use.
|
|
1206
|
+
libraries : Dict[str, str], default {}
|
|
1207
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1208
|
+
python : str, optional, default None
|
|
1209
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1210
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1211
|
+
disabled : bool, default False
|
|
1212
|
+
If set to True, disables Conda.
|
|
1042
1213
|
"""
|
|
1043
1214
|
...
|
|
1044
1215
|
|
|
@@ -1178,173 +1349,3 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
|
1178
1349
|
"""
|
|
1179
1350
|
...
|
|
1180
1351
|
|
|
1181
|
-
def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1182
|
-
"""
|
|
1183
|
-
Specifies what flows belong to the same project.
|
|
1184
|
-
|
|
1185
|
-
A project-specific namespace is created for all flows that
|
|
1186
|
-
use the same `@project(name)`.
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
Parameters
|
|
1190
|
-
----------
|
|
1191
|
-
name : str
|
|
1192
|
-
Project name. Make sure that the name is unique amongst all
|
|
1193
|
-
projects that use the same production scheduler. The name may
|
|
1194
|
-
contain only lowercase alphanumeric characters and underscores.
|
|
1195
|
-
|
|
1196
|
-
branch : Optional[str], default None
|
|
1197
|
-
The branch to use. If not specified, the branch is set to
|
|
1198
|
-
`user.<username>` unless `production` is set to `True`. This can
|
|
1199
|
-
also be set on the command line using `--branch` as a top-level option.
|
|
1200
|
-
It is an error to specify `branch` in the decorator and on the command line.
|
|
1201
|
-
|
|
1202
|
-
production : bool, default False
|
|
1203
|
-
Whether or not the branch is the production branch. This can also be set on the
|
|
1204
|
-
command line using `--production` as a top-level option. It is an error to specify
|
|
1205
|
-
`production` in the decorator and on the command line.
|
|
1206
|
-
The project branch name will be:
|
|
1207
|
-
- if `branch` is specified:
|
|
1208
|
-
- if `production` is True: `prod.<branch>`
|
|
1209
|
-
- if `production` is False: `test.<branch>`
|
|
1210
|
-
- if `branch` is not specified:
|
|
1211
|
-
- if `production` is True: `prod`
|
|
1212
|
-
- if `production` is False: `user.<username>`
|
|
1213
|
-
"""
|
|
1214
|
-
...
|
|
1215
|
-
|
|
1216
|
-
@typing.overload
|
|
1217
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1218
|
-
"""
|
|
1219
|
-
Specifies the PyPI packages for all steps of the flow.
|
|
1220
|
-
|
|
1221
|
-
Use `@pypi_base` to set common packages required by all
|
|
1222
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
1223
|
-
|
|
1224
|
-
Parameters
|
|
1225
|
-
----------
|
|
1226
|
-
packages : Dict[str, str], default: {}
|
|
1227
|
-
Packages to use for this flow. The key is the name of the package
|
|
1228
|
-
and the value is the version to use.
|
|
1229
|
-
python : str, optional, default: None
|
|
1230
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1231
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1232
|
-
"""
|
|
1233
|
-
...
|
|
1234
|
-
|
|
1235
|
-
@typing.overload
|
|
1236
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1237
|
-
...
|
|
1238
|
-
|
|
1239
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
|
1240
|
-
"""
|
|
1241
|
-
Specifies the PyPI packages for all steps of the flow.
|
|
1242
|
-
|
|
1243
|
-
Use `@pypi_base` to set common packages required by all
|
|
1244
|
-
steps and use `@pypi` to specify step-specific overrides.
|
|
1245
|
-
|
|
1246
|
-
Parameters
|
|
1247
|
-
----------
|
|
1248
|
-
packages : Dict[str, str], default: {}
|
|
1249
|
-
Packages to use for this flow. The key is the name of the package
|
|
1250
|
-
and the value is the version to use.
|
|
1251
|
-
python : str, optional, default: None
|
|
1252
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1253
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1254
|
-
"""
|
|
1255
|
-
...
|
|
1256
|
-
|
|
1257
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1258
|
-
"""
|
|
1259
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
|
1260
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
Parameters
|
|
1264
|
-
----------
|
|
1265
|
-
timeout : int
|
|
1266
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
|
1267
|
-
poke_interval : int
|
|
1268
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
|
1269
|
-
mode : str
|
|
1270
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
|
1271
|
-
exponential_backoff : bool
|
|
1272
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
|
1273
|
-
pool : str
|
|
1274
|
-
the slot pool this task should run in,
|
|
1275
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
|
1276
|
-
soft_fail : bool
|
|
1277
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
|
1278
|
-
name : str
|
|
1279
|
-
Name of the sensor on Airflow
|
|
1280
|
-
description : str
|
|
1281
|
-
Description of sensor in the Airflow UI
|
|
1282
|
-
external_dag_id : str
|
|
1283
|
-
The dag_id that contains the task you want to wait for.
|
|
1284
|
-
external_task_ids : List[str]
|
|
1285
|
-
The list of task_ids that you want to wait for.
|
|
1286
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
|
1287
|
-
allowed_states : List[str]
|
|
1288
|
-
Iterable of allowed states, (Default: ['success'])
|
|
1289
|
-
failed_states : List[str]
|
|
1290
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
|
1291
|
-
execution_delta : datetime.timedelta
|
|
1292
|
-
time difference with the previous execution to look at,
|
|
1293
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
|
1294
|
-
check_existence: bool
|
|
1295
|
-
Set to True to check if the external task exists or check if
|
|
1296
|
-
the DAG to wait for exists. (Default: True)
|
|
1297
|
-
"""
|
|
1298
|
-
...
|
|
1299
|
-
|
|
1300
|
-
@typing.overload
|
|
1301
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
|
1302
|
-
"""
|
|
1303
|
-
Specifies the Conda environment for all steps of the flow.
|
|
1304
|
-
|
|
1305
|
-
Use `@conda_base` to set common libraries required by all
|
|
1306
|
-
steps and use `@conda` to specify step-specific additions.
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
Parameters
|
|
1310
|
-
----------
|
|
1311
|
-
packages : Dict[str, str], default {}
|
|
1312
|
-
Packages to use for this flow. The key is the name of the package
|
|
1313
|
-
and the value is the version to use.
|
|
1314
|
-
libraries : Dict[str, str], default {}
|
|
1315
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1316
|
-
python : str, optional, default None
|
|
1317
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1318
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1319
|
-
disabled : bool, default False
|
|
1320
|
-
If set to True, disables Conda.
|
|
1321
|
-
"""
|
|
1322
|
-
...
|
|
1323
|
-
|
|
1324
|
-
@typing.overload
|
|
1325
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
|
1326
|
-
...
|
|
1327
|
-
|
|
1328
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
|
1329
|
-
"""
|
|
1330
|
-
Specifies the Conda environment for all steps of the flow.
|
|
1331
|
-
|
|
1332
|
-
Use `@conda_base` to set common libraries required by all
|
|
1333
|
-
steps and use `@conda` to specify step-specific additions.
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
Parameters
|
|
1337
|
-
----------
|
|
1338
|
-
packages : Dict[str, str], default {}
|
|
1339
|
-
Packages to use for this flow. The key is the name of the package
|
|
1340
|
-
and the value is the version to use.
|
|
1341
|
-
libraries : Dict[str, str], default {}
|
|
1342
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
|
1343
|
-
python : str, optional, default None
|
|
1344
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
|
1345
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
|
1346
|
-
disabled : bool, default False
|
|
1347
|
-
If set to True, disables Conda.
|
|
1348
|
-
"""
|
|
1349
|
-
...
|
|
1350
|
-
|