metaflow-stubs 2.12.38__py2.py3-none-any.whl → 2.13__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +222 -221
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/cli_components/__init__.pyi +2 -2
- metaflow-stubs/cli_components/utils.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +3 -3
- metaflow-stubs/flowspec.pyi +6 -6
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +4 -4
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +4 -2
- metaflow-stubs/metaflow_current.pyi +23 -23
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +12 -12
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +30 -2
- metaflow-stubs/plugins/aws/__init__.pyi +3 -3
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -13
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +4 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +6 -6
- metaflow-stubs/runner/deployer_impl.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +4 -4
- metaflow-stubs/runner/nbdeploy.pyi +4 -2
- metaflow-stubs/runner/nbrun.pyi +3 -3
- metaflow-stubs/runner/subprocess_manager.pyi +5 -2
- metaflow-stubs/runner/utils.pyi +8 -8
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/user_configs/__init__.pyi +2 -2
- metaflow-stubs/user_configs/config_decorators.pyi +5 -5
- metaflow-stubs/user_configs/config_options.pyi +8 -3
- metaflow-stubs/user_configs/config_parameters.pyi +8 -8
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.12.38.dist-info → metaflow_stubs-2.13.dist-info}/METADATA +2 -2
- metaflow_stubs-2.13.dist-info/RECORD +144 -0
- metaflow_stubs-2.12.38.dist-info/RECORD +0 -144
- {metaflow_stubs-2.12.38.dist-info → metaflow_stubs-2.13.dist-info}/WHEEL +0 -0
- {metaflow_stubs-2.12.38.dist-info → metaflow_stubs-2.13.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.
|
4
|
-
# Generated on 2024-12-
|
3
|
+
# MF version: 2.13 #
|
4
|
+
# Generated on 2024-12-20T07:38:30.145823 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import typing
|
12
11
|
import datetime
|
12
|
+
import typing
|
13
13
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
14
14
|
StepFlag = typing.NewType("StepFlag", bool)
|
15
15
|
|
@@ -31,6 +31,7 @@ from .parameters import Parameter as Parameter
|
|
31
31
|
from .parameters import JSONTypeClass as JSONTypeClass
|
32
32
|
from .parameters import JSONType as JSONType
|
33
33
|
from .user_configs.config_parameters import Config as Config
|
34
|
+
from .user_configs.config_parameters import ConfigValue as ConfigValue
|
34
35
|
from .user_configs.config_parameters import config_expr as config_expr
|
35
36
|
from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
|
36
37
|
from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
|
@@ -141,115 +142,94 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
141
142
|
"""
|
142
143
|
...
|
143
144
|
|
144
|
-
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
145
|
-
"""
|
146
|
-
Specifies that this step should execute on Kubernetes.
|
147
|
-
"""
|
148
|
-
...
|
149
|
-
|
150
145
|
@typing.overload
|
151
|
-
def
|
146
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
152
147
|
"""
|
153
|
-
Specifies
|
148
|
+
Specifies a timeout for your step.
|
154
149
|
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
150
|
+
This decorator is useful if this step may hang indefinitely.
|
151
|
+
|
152
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
153
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
154
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
155
|
+
|
156
|
+
Note that all the values specified in parameters are added together so if you specify
|
157
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
159
158
|
"""
|
160
159
|
...
|
161
160
|
|
162
161
|
@typing.overload
|
163
|
-
def
|
162
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
164
163
|
...
|
165
164
|
|
166
165
|
@typing.overload
|
167
|
-
def
|
166
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
168
167
|
...
|
169
168
|
|
170
|
-
def
|
169
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
171
170
|
"""
|
172
|
-
Specifies
|
171
|
+
Specifies a timeout for your step.
|
173
172
|
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
173
|
+
This decorator is useful if this step may hang indefinitely.
|
174
|
+
|
175
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
176
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
177
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
178
|
+
|
179
|
+
Note that all the values specified in parameters are added together so if you specify
|
180
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
178
181
|
"""
|
179
182
|
...
|
180
183
|
|
181
184
|
@typing.overload
|
182
|
-
def
|
185
|
+
def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
183
186
|
"""
|
184
|
-
Specifies
|
185
|
-
|
186
|
-
Information in this decorator will augment any
|
187
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
188
|
-
you can use `@conda_base` to set packages required by all
|
189
|
-
steps and use `@conda` to specify step-specific overrides.
|
187
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
190
188
|
"""
|
191
189
|
...
|
192
190
|
|
193
191
|
@typing.overload
|
194
|
-
def
|
192
|
+
def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
195
193
|
...
|
196
194
|
|
197
195
|
@typing.overload
|
198
|
-
def
|
196
|
+
def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
199
197
|
...
|
200
198
|
|
201
|
-
def
|
199
|
+
def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
|
202
200
|
"""
|
203
|
-
Specifies
|
204
|
-
|
205
|
-
Information in this decorator will augment any
|
206
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
207
|
-
you can use `@conda_base` to set packages required by all
|
208
|
-
steps and use `@conda` to specify step-specific overrides.
|
201
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
209
202
|
"""
|
210
203
|
...
|
211
204
|
|
212
205
|
@typing.overload
|
213
|
-
def
|
206
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
214
207
|
"""
|
215
|
-
|
208
|
+
Specifies the PyPI packages for the step.
|
216
209
|
|
217
|
-
|
210
|
+
Information in this decorator will augment any
|
211
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
212
|
+
you can use `@pypi_base` to set packages required by all
|
213
|
+
steps and use `@pypi` to specify step-specific overrides.
|
218
214
|
"""
|
219
215
|
...
|
220
216
|
|
221
217
|
@typing.overload
|
222
|
-
def
|
218
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
223
219
|
...
|
224
220
|
|
225
221
|
@typing.overload
|
226
|
-
def
|
222
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
227
223
|
...
|
228
224
|
|
229
|
-
def
|
225
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
230
226
|
"""
|
231
|
-
|
227
|
+
Specifies the PyPI packages for the step.
|
232
228
|
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
@typing.overload
|
238
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
239
|
-
"""
|
240
|
-
Decorator prototype for all step decorators. This function gets specialized
|
241
|
-
and imported for all decorators types by _import_plugin_decorators().
|
242
|
-
"""
|
243
|
-
...
|
244
|
-
|
245
|
-
@typing.overload
|
246
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
247
|
-
...
|
248
|
-
|
249
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
250
|
-
"""
|
251
|
-
Decorator prototype for all step decorators. This function gets specialized
|
252
|
-
and imported for all decorators types by _import_plugin_decorators().
|
229
|
+
Information in this decorator will augment any
|
230
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
231
|
+
you can use `@pypi_base` to set packages required by all
|
232
|
+
steps and use `@pypi` to specify step-specific overrides.
|
253
233
|
"""
|
254
234
|
...
|
255
235
|
|
@@ -274,29 +254,6 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
|
|
274
254
|
"""
|
275
255
|
...
|
276
256
|
|
277
|
-
@typing.overload
|
278
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
279
|
-
"""
|
280
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
281
|
-
the execution of a step.
|
282
|
-
"""
|
283
|
-
...
|
284
|
-
|
285
|
-
@typing.overload
|
286
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
287
|
-
...
|
288
|
-
|
289
|
-
@typing.overload
|
290
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
291
|
-
...
|
292
|
-
|
293
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
294
|
-
"""
|
295
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
296
|
-
the execution of a step.
|
297
|
-
"""
|
298
|
-
...
|
299
|
-
|
300
257
|
@typing.overload
|
301
258
|
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
302
259
|
"""
|
@@ -336,6 +293,25 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
336
293
|
"""
|
337
294
|
...
|
338
295
|
|
296
|
+
@typing.overload
|
297
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
298
|
+
"""
|
299
|
+
Decorator prototype for all step decorators. This function gets specialized
|
300
|
+
and imported for all decorators types by _import_plugin_decorators().
|
301
|
+
"""
|
302
|
+
...
|
303
|
+
|
304
|
+
@typing.overload
|
305
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
306
|
+
...
|
307
|
+
|
308
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
309
|
+
"""
|
310
|
+
Decorator prototype for all step decorators. This function gets specialized
|
311
|
+
and imported for all decorators types by _import_plugin_decorators().
|
312
|
+
"""
|
313
|
+
...
|
314
|
+
|
339
315
|
@typing.overload
|
340
316
|
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
341
317
|
"""
|
@@ -386,133 +362,191 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
386
362
|
...
|
387
363
|
|
388
364
|
@typing.overload
|
389
|
-
def
|
365
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
390
366
|
"""
|
391
|
-
Specifies
|
367
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
368
|
+
the execution of a step.
|
392
369
|
"""
|
393
370
|
...
|
394
371
|
|
395
372
|
@typing.overload
|
396
|
-
def
|
373
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
397
374
|
...
|
398
375
|
|
399
376
|
@typing.overload
|
400
|
-
def
|
377
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
401
378
|
...
|
402
379
|
|
403
|
-
def
|
380
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
404
381
|
"""
|
405
|
-
Specifies
|
382
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
383
|
+
the execution of a step.
|
384
|
+
"""
|
385
|
+
...
|
386
|
+
|
387
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
388
|
+
"""
|
389
|
+
Specifies that this step should execute on Kubernetes.
|
406
390
|
"""
|
407
391
|
...
|
408
392
|
|
409
393
|
@typing.overload
|
410
|
-
def
|
394
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
411
395
|
"""
|
412
|
-
Specifies
|
396
|
+
Specifies the Conda environment for the step.
|
413
397
|
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
398
|
+
Information in this decorator will augment any
|
399
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
400
|
+
you can use `@conda_base` to set packages required by all
|
401
|
+
steps and use `@conda` to specify step-specific overrides.
|
418
402
|
"""
|
419
403
|
...
|
420
404
|
|
421
405
|
@typing.overload
|
422
|
-
def
|
406
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
423
407
|
...
|
424
408
|
|
425
409
|
@typing.overload
|
426
|
-
def
|
410
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
427
411
|
...
|
428
412
|
|
429
|
-
def
|
413
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
430
414
|
"""
|
431
|
-
Specifies
|
415
|
+
Specifies the Conda environment for the step.
|
432
416
|
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
417
|
+
Information in this decorator will augment any
|
418
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
419
|
+
you can use `@conda_base` to set packages required by all
|
420
|
+
steps and use `@conda` to specify step-specific overrides.
|
437
421
|
"""
|
438
422
|
...
|
439
423
|
|
440
424
|
@typing.overload
|
441
|
-
def
|
425
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
442
426
|
"""
|
443
|
-
|
444
|
-
|
445
|
-
This decorator is useful if this step may hang indefinitely.
|
446
|
-
|
447
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
448
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
449
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
427
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
450
428
|
|
451
|
-
Note that
|
452
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
429
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
453
430
|
"""
|
454
431
|
...
|
455
432
|
|
456
433
|
@typing.overload
|
457
|
-
def
|
434
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
458
435
|
...
|
459
436
|
|
460
437
|
@typing.overload
|
461
|
-
def
|
438
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
462
439
|
...
|
463
440
|
|
464
|
-
def
|
441
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
465
442
|
"""
|
466
|
-
|
467
|
-
|
468
|
-
This decorator is useful if this step may hang indefinitely.
|
469
|
-
|
470
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
471
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
472
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
443
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
473
444
|
|
474
|
-
Note that
|
475
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
445
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
476
446
|
"""
|
477
447
|
...
|
478
448
|
|
479
449
|
@typing.overload
|
480
|
-
def
|
450
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
481
451
|
"""
|
482
|
-
Specifies the
|
452
|
+
Specifies that the step will success under all circumstances.
|
483
453
|
|
484
|
-
|
485
|
-
|
454
|
+
The decorator will create an optional artifact, specified by `var`, which
|
455
|
+
contains the exception raised. You can use it to detect the presence
|
456
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
457
|
+
are missing.
|
486
458
|
"""
|
487
459
|
...
|
488
460
|
|
489
461
|
@typing.overload
|
490
|
-
def
|
462
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
491
463
|
...
|
492
464
|
|
493
|
-
|
465
|
+
@typing.overload
|
466
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
467
|
+
...
|
468
|
+
|
469
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
494
470
|
"""
|
495
|
-
Specifies the
|
471
|
+
Specifies that the step will success under all circumstances.
|
496
472
|
|
497
|
-
|
498
|
-
|
473
|
+
The decorator will create an optional artifact, specified by `var`, which
|
474
|
+
contains the exception raised. You can use it to detect the presence
|
475
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
476
|
+
are missing.
|
499
477
|
"""
|
500
478
|
...
|
501
479
|
|
502
|
-
|
480
|
+
@typing.overload
|
481
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
503
482
|
"""
|
504
|
-
|
505
|
-
|
483
|
+
Specifies the event(s) that this flow depends on.
|
484
|
+
|
485
|
+
```
|
486
|
+
@trigger(event='foo')
|
487
|
+
```
|
488
|
+
or
|
489
|
+
```
|
490
|
+
@trigger(events=['foo', 'bar'])
|
491
|
+
```
|
492
|
+
|
493
|
+
Additionally, you can specify the parameter mappings
|
494
|
+
to map event payload to Metaflow parameters for the flow.
|
495
|
+
```
|
496
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
497
|
+
```
|
498
|
+
or
|
499
|
+
```
|
500
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
501
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
502
|
+
```
|
503
|
+
|
504
|
+
'parameters' can also be a list of strings and tuples like so:
|
505
|
+
```
|
506
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
507
|
+
```
|
508
|
+
This is equivalent to:
|
509
|
+
```
|
510
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
511
|
+
```
|
506
512
|
"""
|
507
513
|
...
|
508
514
|
|
509
|
-
|
515
|
+
@typing.overload
|
516
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
517
|
+
...
|
518
|
+
|
519
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
510
520
|
"""
|
511
|
-
|
512
|
-
|
513
|
-
|
514
|
-
|
515
|
-
|
521
|
+
Specifies the event(s) that this flow depends on.
|
522
|
+
|
523
|
+
```
|
524
|
+
@trigger(event='foo')
|
525
|
+
```
|
526
|
+
or
|
527
|
+
```
|
528
|
+
@trigger(events=['foo', 'bar'])
|
529
|
+
```
|
530
|
+
|
531
|
+
Additionally, you can specify the parameter mappings
|
532
|
+
to map event payload to Metaflow parameters for the flow.
|
533
|
+
```
|
534
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
535
|
+
```
|
536
|
+
or
|
537
|
+
```
|
538
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
539
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
540
|
+
```
|
541
|
+
|
542
|
+
'parameters' can also be a list of strings and tuples like so:
|
543
|
+
```
|
544
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
545
|
+
```
|
546
|
+
This is equivalent to:
|
547
|
+
```
|
548
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
549
|
+
```
|
516
550
|
"""
|
517
551
|
...
|
518
552
|
|
@@ -597,6 +631,25 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
597
631
|
"""
|
598
632
|
...
|
599
633
|
|
634
|
+
@typing.overload
|
635
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
636
|
+
"""
|
637
|
+
Specifies the times when the flow should be run when running on a
|
638
|
+
production scheduler.
|
639
|
+
"""
|
640
|
+
...
|
641
|
+
|
642
|
+
@typing.overload
|
643
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
644
|
+
...
|
645
|
+
|
646
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
647
|
+
"""
|
648
|
+
Specifies the times when the flow should be run when running on a
|
649
|
+
production scheduler.
|
650
|
+
"""
|
651
|
+
...
|
652
|
+
|
600
653
|
@typing.overload
|
601
654
|
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
602
655
|
"""
|
@@ -621,94 +674,42 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
|
|
621
674
|
...
|
622
675
|
|
623
676
|
@typing.overload
|
624
|
-
def
|
677
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
625
678
|
"""
|
626
|
-
Specifies the
|
627
|
-
|
679
|
+
Specifies the PyPI packages for all steps of the flow.
|
680
|
+
|
681
|
+
Use `@pypi_base` to set common packages required by all
|
682
|
+
steps and use `@pypi` to specify step-specific overrides.
|
628
683
|
"""
|
629
684
|
...
|
630
685
|
|
631
686
|
@typing.overload
|
632
|
-
def
|
687
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
633
688
|
...
|
634
689
|
|
635
|
-
def
|
690
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
636
691
|
"""
|
637
|
-
Specifies the
|
638
|
-
|
692
|
+
Specifies the PyPI packages for all steps of the flow.
|
693
|
+
|
694
|
+
Use `@pypi_base` to set common packages required by all
|
695
|
+
steps and use `@pypi` to specify step-specific overrides.
|
639
696
|
"""
|
640
697
|
...
|
641
698
|
|
642
|
-
|
643
|
-
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
699
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
644
700
|
"""
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
or
|
651
|
-
```
|
652
|
-
@trigger(events=['foo', 'bar'])
|
653
|
-
```
|
654
|
-
|
655
|
-
Additionally, you can specify the parameter mappings
|
656
|
-
to map event payload to Metaflow parameters for the flow.
|
657
|
-
```
|
658
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
659
|
-
```
|
660
|
-
or
|
661
|
-
```
|
662
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
663
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
664
|
-
```
|
665
|
-
|
666
|
-
'parameters' can also be a list of strings and tuples like so:
|
667
|
-
```
|
668
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
669
|
-
```
|
670
|
-
This is equivalent to:
|
671
|
-
```
|
672
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
673
|
-
```
|
701
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
702
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
703
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
704
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
705
|
+
starts only after all sensors finish.
|
674
706
|
"""
|
675
707
|
...
|
676
708
|
|
677
|
-
|
678
|
-
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
679
|
-
...
|
680
|
-
|
681
|
-
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
709
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
682
710
|
"""
|
683
|
-
|
684
|
-
|
685
|
-
```
|
686
|
-
@trigger(event='foo')
|
687
|
-
```
|
688
|
-
or
|
689
|
-
```
|
690
|
-
@trigger(events=['foo', 'bar'])
|
691
|
-
```
|
692
|
-
|
693
|
-
Additionally, you can specify the parameter mappings
|
694
|
-
to map event payload to Metaflow parameters for the flow.
|
695
|
-
```
|
696
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
697
|
-
```
|
698
|
-
or
|
699
|
-
```
|
700
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
701
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
702
|
-
```
|
703
|
-
|
704
|
-
'parameters' can also be a list of strings and tuples like so:
|
705
|
-
```
|
706
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
707
|
-
```
|
708
|
-
This is equivalent to:
|
709
|
-
```
|
710
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
711
|
-
```
|
711
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
712
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
712
713
|
"""
|
713
714
|
...
|
714
715
|
|