metaflow-stubs 2.12.32__py2.py3-none-any.whl → 2.12.34__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +246 -246
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +3 -3
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +2 -2
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +18 -18
- metaflow-stubs/mflog/__init__.pyi +2 -2
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +11 -11
- metaflow-stubs/parameters.pyi +2 -2
- metaflow-stubs/plugins/__init__.pyi +12 -12
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -3
- metaflow-stubs/plugins/aws/__init__.pyi +3 -3
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +3 -3
- metaflow-stubs/runner/deployer.pyi +5 -5
- metaflow-stubs/runner/deployer_impl.pyi +3 -2
- metaflow-stubs/runner/metaflow_runner.pyi +8 -4
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +7 -3
- metaflow-stubs/runner/utils.pyi +108 -9
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.12.32.dist-info → metaflow_stubs-2.12.34.dist-info}/METADATA +2 -2
- metaflow_stubs-2.12.34.dist-info/RECORD +158 -0
- metaflow_stubs-2.12.32.dist-info/RECORD +0 -158
- {metaflow_stubs-2.12.32.dist-info → metaflow_stubs-2.12.34.dist-info}/WHEEL +0 -0
- {metaflow_stubs-2.12.32.dist-info → metaflow_stubs-2.12.34.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.34 #
|
4
|
+
# Generated on 2024-12-04T14:12:34.722470 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -156,12 +156,6 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
156
156
|
"""
|
157
157
|
...
|
158
158
|
|
159
|
-
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
160
|
-
"""
|
161
|
-
Specifies that this step should execute on Kubernetes.
|
162
|
-
"""
|
163
|
-
...
|
164
|
-
|
165
159
|
@typing.overload
|
166
160
|
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
167
161
|
"""
|
@@ -184,90 +178,97 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
|
|
184
178
|
...
|
185
179
|
|
186
180
|
@typing.overload
|
187
|
-
def
|
181
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
188
182
|
"""
|
189
|
-
Specifies
|
190
|
-
|
191
|
-
This decorator is useful if this step may hang indefinitely.
|
183
|
+
Specifies the number of times the task corresponding
|
184
|
+
to a step needs to be retried.
|
192
185
|
|
193
|
-
This
|
194
|
-
|
195
|
-
|
186
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
187
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
188
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
196
189
|
|
197
|
-
|
198
|
-
|
190
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
191
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
192
|
+
ensuring that the flow execution can continue.
|
199
193
|
"""
|
200
194
|
...
|
201
195
|
|
202
196
|
@typing.overload
|
203
|
-
def
|
197
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
204
198
|
...
|
205
199
|
|
206
200
|
@typing.overload
|
207
|
-
def
|
201
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
208
202
|
...
|
209
203
|
|
210
|
-
def
|
204
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
211
205
|
"""
|
212
|
-
Specifies
|
206
|
+
Specifies the number of times the task corresponding
|
207
|
+
to a step needs to be retried.
|
213
208
|
|
214
|
-
This decorator is useful
|
209
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
210
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
211
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
215
212
|
|
216
|
-
This can be used in conjunction with the `@
|
217
|
-
|
218
|
-
|
213
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
214
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
215
|
+
ensuring that the flow execution can continue.
|
216
|
+
"""
|
217
|
+
...
|
218
|
+
|
219
|
+
@typing.overload
|
220
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
221
|
+
"""
|
222
|
+
Specifies that the step will success under all circumstances.
|
219
223
|
|
220
|
-
|
221
|
-
|
224
|
+
The decorator will create an optional artifact, specified by `var`, which
|
225
|
+
contains the exception raised. You can use it to detect the presence
|
226
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
227
|
+
are missing.
|
222
228
|
"""
|
223
229
|
...
|
224
230
|
|
225
231
|
@typing.overload
|
226
|
-
def
|
232
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
233
|
+
...
|
234
|
+
|
235
|
+
@typing.overload
|
236
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
237
|
+
...
|
238
|
+
|
239
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
227
240
|
"""
|
228
|
-
Specifies the
|
241
|
+
Specifies that the step will success under all circumstances.
|
229
242
|
|
230
|
-
|
231
|
-
|
243
|
+
The decorator will create an optional artifact, specified by `var`, which
|
244
|
+
contains the exception raised. You can use it to detect the presence
|
245
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
246
|
+
are missing.
|
247
|
+
"""
|
248
|
+
...
|
249
|
+
|
250
|
+
@typing.overload
|
251
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
252
|
+
"""
|
253
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
232
254
|
|
233
|
-
|
234
|
-
```
|
235
|
-
python myflow.py run --with batch
|
236
|
-
```
|
237
|
-
or
|
238
|
-
```
|
239
|
-
python myflow.py run --with kubernetes
|
240
|
-
```
|
241
|
-
which executes the flow on the desired system using the
|
242
|
-
requirements specified in `@resources`.
|
255
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
243
256
|
"""
|
244
257
|
...
|
245
258
|
|
246
259
|
@typing.overload
|
247
|
-
def
|
260
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
248
261
|
...
|
249
262
|
|
250
263
|
@typing.overload
|
251
|
-
def
|
264
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
252
265
|
...
|
253
266
|
|
254
|
-
def
|
267
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
255
268
|
"""
|
256
|
-
|
257
|
-
|
258
|
-
Use `@resources` to specify the resource requirements
|
259
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
269
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
260
270
|
|
261
|
-
|
262
|
-
```
|
263
|
-
python myflow.py run --with batch
|
264
|
-
```
|
265
|
-
or
|
266
|
-
```
|
267
|
-
python myflow.py run --with kubernetes
|
268
|
-
```
|
269
|
-
which executes the flow on the desired system using the
|
270
|
-
requirements specified in `@resources`.
|
271
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
271
272
|
"""
|
272
273
|
...
|
273
274
|
|
@@ -316,189 +317,258 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
316
317
|
...
|
317
318
|
|
318
319
|
@typing.overload
|
319
|
-
def
|
320
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
320
321
|
"""
|
321
|
-
Specifies
|
322
|
+
Specifies the resources needed when executing this step.
|
322
323
|
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
324
|
+
Use `@resources` to specify the resource requirements
|
325
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
326
|
+
|
327
|
+
You can choose the compute layer on the command line by executing e.g.
|
328
|
+
```
|
329
|
+
python myflow.py run --with batch
|
330
|
+
```
|
331
|
+
or
|
332
|
+
```
|
333
|
+
python myflow.py run --with kubernetes
|
334
|
+
```
|
335
|
+
which executes the flow on the desired system using the
|
336
|
+
requirements specified in `@resources`.
|
327
337
|
"""
|
328
338
|
...
|
329
339
|
|
330
340
|
@typing.overload
|
331
|
-
def
|
341
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
332
342
|
...
|
333
343
|
|
334
344
|
@typing.overload
|
335
|
-
def
|
345
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
336
346
|
...
|
337
347
|
|
338
|
-
def
|
348
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
339
349
|
"""
|
340
|
-
Specifies
|
350
|
+
Specifies the resources needed when executing this step.
|
341
351
|
|
342
|
-
|
343
|
-
|
344
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
345
|
-
are missing.
|
346
|
-
"""
|
347
|
-
...
|
348
|
-
|
349
|
-
@typing.overload
|
350
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
351
|
-
"""
|
352
|
-
Specifies the Conda environment for the step.
|
352
|
+
Use `@resources` to specify the resource requirements
|
353
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
353
354
|
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
355
|
+
You can choose the compute layer on the command line by executing e.g.
|
356
|
+
```
|
357
|
+
python myflow.py run --with batch
|
358
|
+
```
|
359
|
+
or
|
360
|
+
```
|
361
|
+
python myflow.py run --with kubernetes
|
362
|
+
```
|
363
|
+
which executes the flow on the desired system using the
|
364
|
+
requirements specified in `@resources`.
|
358
365
|
"""
|
359
366
|
...
|
360
367
|
|
361
|
-
|
362
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
363
|
-
...
|
364
|
-
|
365
|
-
@typing.overload
|
366
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
367
|
-
...
|
368
|
-
|
369
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
368
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
370
369
|
"""
|
371
|
-
Specifies
|
372
|
-
|
373
|
-
Information in this decorator will augment any
|
374
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
375
|
-
you can use `@conda_base` to set packages required by all
|
376
|
-
steps and use `@conda` to specify step-specific overrides.
|
370
|
+
Specifies that this step should execute on Kubernetes.
|
377
371
|
"""
|
378
372
|
...
|
379
373
|
|
380
374
|
@typing.overload
|
381
|
-
def
|
375
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
382
376
|
"""
|
383
|
-
Specifies
|
384
|
-
to a step needs to be retried.
|
377
|
+
Specifies a timeout for your step.
|
385
378
|
|
386
|
-
This decorator is useful
|
387
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
388
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
379
|
+
This decorator is useful if this step may hang indefinitely.
|
389
380
|
|
390
|
-
This can be used in conjunction with the `@
|
391
|
-
|
392
|
-
|
381
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
382
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
383
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
384
|
+
|
385
|
+
Note that all the values specified in parameters are added together so if you specify
|
386
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
393
387
|
"""
|
394
388
|
...
|
395
389
|
|
396
390
|
@typing.overload
|
397
|
-
def
|
391
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
398
392
|
...
|
399
393
|
|
400
394
|
@typing.overload
|
401
|
-
def
|
395
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
402
396
|
...
|
403
397
|
|
404
|
-
def
|
398
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
405
399
|
"""
|
406
|
-
Specifies
|
407
|
-
to a step needs to be retried.
|
400
|
+
Specifies a timeout for your step.
|
408
401
|
|
409
|
-
This decorator is useful
|
410
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
411
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
402
|
+
This decorator is useful if this step may hang indefinitely.
|
412
403
|
|
413
|
-
This can be used in conjunction with the `@
|
414
|
-
|
415
|
-
|
404
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
405
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
406
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
407
|
+
|
408
|
+
Note that all the values specified in parameters are added together so if you specify
|
409
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
416
410
|
"""
|
417
411
|
...
|
418
412
|
|
419
413
|
@typing.overload
|
420
|
-
def
|
414
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
421
415
|
"""
|
422
|
-
|
416
|
+
Specifies the PyPI packages for the step.
|
423
417
|
|
424
|
-
|
418
|
+
Information in this decorator will augment any
|
419
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
420
|
+
you can use `@pypi_base` to set packages required by all
|
421
|
+
steps and use `@pypi` to specify step-specific overrides.
|
425
422
|
"""
|
426
423
|
...
|
427
424
|
|
428
425
|
@typing.overload
|
429
|
-
def
|
426
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
430
427
|
...
|
431
428
|
|
432
429
|
@typing.overload
|
433
|
-
def
|
430
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
434
431
|
...
|
435
432
|
|
436
|
-
def
|
433
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
437
434
|
"""
|
438
|
-
|
435
|
+
Specifies the PyPI packages for the step.
|
439
436
|
|
440
|
-
|
437
|
+
Information in this decorator will augment any
|
438
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
439
|
+
you can use `@pypi_base` to set packages required by all
|
440
|
+
steps and use `@pypi` to specify step-specific overrides.
|
441
441
|
"""
|
442
442
|
...
|
443
443
|
|
444
444
|
@typing.overload
|
445
|
-
def
|
445
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
446
446
|
"""
|
447
|
-
Specifies the
|
447
|
+
Specifies the Conda environment for the step.
|
448
448
|
|
449
449
|
Information in this decorator will augment any
|
450
|
-
attributes set in the `@
|
451
|
-
you can use `@
|
452
|
-
steps and use `@
|
450
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
451
|
+
you can use `@conda_base` to set packages required by all
|
452
|
+
steps and use `@conda` to specify step-specific overrides.
|
453
453
|
"""
|
454
454
|
...
|
455
455
|
|
456
456
|
@typing.overload
|
457
|
-
def
|
457
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
458
458
|
...
|
459
459
|
|
460
460
|
@typing.overload
|
461
|
-
def
|
461
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
462
462
|
...
|
463
463
|
|
464
|
-
def
|
464
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
465
465
|
"""
|
466
|
-
Specifies the
|
466
|
+
Specifies the Conda environment for the step.
|
467
467
|
|
468
468
|
Information in this decorator will augment any
|
469
|
-
attributes set in the `@
|
470
|
-
you can use `@
|
471
|
-
steps and use `@
|
469
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
470
|
+
you can use `@conda_base` to set packages required by all
|
471
|
+
steps and use `@conda` to specify step-specific overrides.
|
472
472
|
"""
|
473
473
|
...
|
474
474
|
|
475
|
-
|
475
|
+
@typing.overload
|
476
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
476
477
|
"""
|
477
|
-
|
478
|
-
|
478
|
+
Specifies the times when the flow should be run when running on a
|
479
|
+
production scheduler.
|
479
480
|
"""
|
480
481
|
...
|
481
482
|
|
482
483
|
@typing.overload
|
483
|
-
def
|
484
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
485
|
+
...
|
486
|
+
|
487
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
484
488
|
"""
|
485
|
-
Specifies the
|
489
|
+
Specifies the times when the flow should be run when running on a
|
490
|
+
production scheduler.
|
491
|
+
"""
|
492
|
+
...
|
493
|
+
|
494
|
+
@typing.overload
|
495
|
+
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
496
|
+
"""
|
497
|
+
Specifies the flow(s) that this flow depends on.
|
486
498
|
|
487
|
-
|
488
|
-
|
499
|
+
```
|
500
|
+
@trigger_on_finish(flow='FooFlow')
|
501
|
+
```
|
502
|
+
or
|
503
|
+
```
|
504
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
505
|
+
```
|
506
|
+
This decorator respects the @project decorator and triggers the flow
|
507
|
+
when upstream runs within the same namespace complete successfully
|
508
|
+
|
509
|
+
Additionally, you can specify project aware upstream flow dependencies
|
510
|
+
by specifying the fully qualified project_flow_name.
|
511
|
+
```
|
512
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
513
|
+
```
|
514
|
+
or
|
515
|
+
```
|
516
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
517
|
+
```
|
518
|
+
|
519
|
+
You can also specify just the project or project branch (other values will be
|
520
|
+
inferred from the current project or project branch):
|
521
|
+
```
|
522
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
523
|
+
```
|
524
|
+
|
525
|
+
Note that `branch` is typically one of:
|
526
|
+
- `prod`
|
527
|
+
- `user.bob`
|
528
|
+
- `test.my_experiment`
|
529
|
+
- `prod.staging`
|
489
530
|
"""
|
490
531
|
...
|
491
532
|
|
492
533
|
@typing.overload
|
493
|
-
def
|
534
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
494
535
|
...
|
495
536
|
|
496
|
-
def
|
537
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
497
538
|
"""
|
498
|
-
Specifies the
|
539
|
+
Specifies the flow(s) that this flow depends on.
|
499
540
|
|
500
|
-
|
501
|
-
|
541
|
+
```
|
542
|
+
@trigger_on_finish(flow='FooFlow')
|
543
|
+
```
|
544
|
+
or
|
545
|
+
```
|
546
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
547
|
+
```
|
548
|
+
This decorator respects the @project decorator and triggers the flow
|
549
|
+
when upstream runs within the same namespace complete successfully
|
550
|
+
|
551
|
+
Additionally, you can specify project aware upstream flow dependencies
|
552
|
+
by specifying the fully qualified project_flow_name.
|
553
|
+
```
|
554
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
555
|
+
```
|
556
|
+
or
|
557
|
+
```
|
558
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
559
|
+
```
|
560
|
+
|
561
|
+
You can also specify just the project or project branch (other values will be
|
562
|
+
inferred from the current project or project branch):
|
563
|
+
```
|
564
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
565
|
+
```
|
566
|
+
|
567
|
+
Note that `branch` is typically one of:
|
568
|
+
- `prod`
|
569
|
+
- `user.bob`
|
570
|
+
- `test.my_experiment`
|
571
|
+
- `prod.staging`
|
502
572
|
"""
|
503
573
|
...
|
504
574
|
|
@@ -511,6 +581,23 @@ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typ
|
|
511
581
|
"""
|
512
582
|
...
|
513
583
|
|
584
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
585
|
+
"""
|
586
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
587
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
588
|
+
"""
|
589
|
+
...
|
590
|
+
|
591
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
592
|
+
"""
|
593
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
594
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
595
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
596
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
597
|
+
starts only after all sensors finish.
|
598
|
+
"""
|
599
|
+
...
|
600
|
+
|
514
601
|
@typing.overload
|
515
602
|
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
516
603
|
"""
|
@@ -608,112 +695,25 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
|
|
608
695
|
...
|
609
696
|
|
610
697
|
@typing.overload
|
611
|
-
def
|
698
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
612
699
|
"""
|
613
|
-
Specifies the
|
614
|
-
|
615
|
-
```
|
616
|
-
@trigger_on_finish(flow='FooFlow')
|
617
|
-
```
|
618
|
-
or
|
619
|
-
```
|
620
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
621
|
-
```
|
622
|
-
This decorator respects the @project decorator and triggers the flow
|
623
|
-
when upstream runs within the same namespace complete successfully
|
624
|
-
|
625
|
-
Additionally, you can specify project aware upstream flow dependencies
|
626
|
-
by specifying the fully qualified project_flow_name.
|
627
|
-
```
|
628
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
629
|
-
```
|
630
|
-
or
|
631
|
-
```
|
632
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
633
|
-
```
|
634
|
-
|
635
|
-
You can also specify just the project or project branch (other values will be
|
636
|
-
inferred from the current project or project branch):
|
637
|
-
```
|
638
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
639
|
-
```
|
700
|
+
Specifies the Conda environment for all steps of the flow.
|
640
701
|
|
641
|
-
|
642
|
-
|
643
|
-
- `user.bob`
|
644
|
-
- `test.my_experiment`
|
645
|
-
- `prod.staging`
|
702
|
+
Use `@conda_base` to set common libraries required by all
|
703
|
+
steps and use `@conda` to specify step-specific additions.
|
646
704
|
"""
|
647
705
|
...
|
648
706
|
|
649
707
|
@typing.overload
|
650
|
-
def
|
708
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
651
709
|
...
|
652
710
|
|
653
|
-
def
|
711
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
654
712
|
"""
|
655
|
-
Specifies the
|
656
|
-
|
657
|
-
```
|
658
|
-
@trigger_on_finish(flow='FooFlow')
|
659
|
-
```
|
660
|
-
or
|
661
|
-
```
|
662
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
663
|
-
```
|
664
|
-
This decorator respects the @project decorator and triggers the flow
|
665
|
-
when upstream runs within the same namespace complete successfully
|
666
|
-
|
667
|
-
Additionally, you can specify project aware upstream flow dependencies
|
668
|
-
by specifying the fully qualified project_flow_name.
|
669
|
-
```
|
670
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
671
|
-
```
|
672
|
-
or
|
673
|
-
```
|
674
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
675
|
-
```
|
676
|
-
|
677
|
-
You can also specify just the project or project branch (other values will be
|
678
|
-
inferred from the current project or project branch):
|
679
|
-
```
|
680
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
681
|
-
```
|
713
|
+
Specifies the Conda environment for all steps of the flow.
|
682
714
|
|
683
|
-
|
684
|
-
|
685
|
-
- `user.bob`
|
686
|
-
- `test.my_experiment`
|
687
|
-
- `prod.staging`
|
688
|
-
"""
|
689
|
-
...
|
690
|
-
|
691
|
-
@typing.overload
|
692
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
693
|
-
"""
|
694
|
-
Specifies the times when the flow should be run when running on a
|
695
|
-
production scheduler.
|
696
|
-
"""
|
697
|
-
...
|
698
|
-
|
699
|
-
@typing.overload
|
700
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
701
|
-
...
|
702
|
-
|
703
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
704
|
-
"""
|
705
|
-
Specifies the times when the flow should be run when running on a
|
706
|
-
production scheduler.
|
707
|
-
"""
|
708
|
-
...
|
709
|
-
|
710
|
-
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
711
|
-
"""
|
712
|
-
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
713
|
-
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
714
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
715
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
716
|
-
starts only after all sensors finish.
|
715
|
+
Use `@conda_base` to set common libraries required by all
|
716
|
+
steps and use `@conda` to specify step-specific additions.
|
717
717
|
"""
|
718
718
|
...
|
719
719
|
|