metaflow-stubs 2.12.33__py2.py3-none-any.whl → 2.12.34__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +250 -250
- metaflow-stubs/cards.pyi +2 -2
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +2 -2
- metaflow-stubs/client/core.pyi +3 -3
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +2 -2
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata_provider/__init__.pyi +2 -2
- metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
- metaflow-stubs/metadata_provider/metadata.pyi +2 -2
- metaflow-stubs/metadata_provider/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +24 -24
- metaflow-stubs/mflog/__init__.pyi +2 -2
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +11 -11
- metaflow-stubs/parameters.pyi +2 -2
- metaflow-stubs/plugins/__init__.pyi +13 -13
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -2
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +2 -2
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +12 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +2 -2
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +3 -3
- metaflow-stubs/runner/deployer.pyi +4 -4
- metaflow-stubs/runner/deployer_impl.pyi +3 -2
- metaflow-stubs/runner/metaflow_runner.pyi +8 -4
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +7 -3
- metaflow-stubs/runner/utils.pyi +108 -9
- metaflow-stubs/system/__init__.pyi +2 -2
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.12.33.dist-info → metaflow_stubs-2.12.34.dist-info}/METADATA +2 -2
- metaflow_stubs-2.12.34.dist-info/RECORD +158 -0
- metaflow_stubs-2.12.33.dist-info/RECORD +0 -158
- {metaflow_stubs-2.12.33.dist-info → metaflow_stubs-2.12.34.dist-info}/WHEEL +0 -0
- {metaflow_stubs-2.12.33.dist-info → metaflow_stubs-2.12.34.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
######################################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.34 #
|
4
|
+
# Generated on 2024-12-04T14:12:34.722470 #
|
5
5
|
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -29,8 +29,8 @@ from .flowspec import FlowSpec as FlowSpec
|
|
29
29
|
from .parameters import Parameter as Parameter
|
30
30
|
from .parameters import JSONTypeClass as JSONTypeClass
|
31
31
|
from .parameters import JSONType as JSONType
|
32
|
-
from . import events as events
|
33
32
|
from . import tuple_util as tuple_util
|
33
|
+
from . import events as events
|
34
34
|
from . import runner as runner
|
35
35
|
from . import plugins as plugins
|
36
36
|
from .plugins.datatools.s3.s3 import S3 as S3
|
@@ -138,66 +138,81 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
138
138
|
...
|
139
139
|
|
140
140
|
@typing.overload
|
141
|
-
def
|
141
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
142
142
|
"""
|
143
|
-
|
144
|
-
|
145
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
143
|
+
Decorator prototype for all step decorators. This function gets specialized
|
144
|
+
and imported for all decorators types by _import_plugin_decorators().
|
146
145
|
"""
|
147
146
|
...
|
148
147
|
|
149
148
|
@typing.overload
|
150
|
-
def
|
149
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
150
|
+
...
|
151
|
+
|
152
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
153
|
+
"""
|
154
|
+
Decorator prototype for all step decorators. This function gets specialized
|
155
|
+
and imported for all decorators types by _import_plugin_decorators().
|
156
|
+
"""
|
151
157
|
...
|
152
158
|
|
153
159
|
@typing.overload
|
154
|
-
def
|
160
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
161
|
+
"""
|
162
|
+
Specifies environment variables to be set prior to the execution of a step.
|
163
|
+
"""
|
155
164
|
...
|
156
165
|
|
157
|
-
|
166
|
+
@typing.overload
|
167
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
168
|
+
...
|
169
|
+
|
170
|
+
@typing.overload
|
171
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
172
|
+
...
|
173
|
+
|
174
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
158
175
|
"""
|
159
|
-
|
160
|
-
|
161
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
176
|
+
Specifies environment variables to be set prior to the execution of a step.
|
162
177
|
"""
|
163
178
|
...
|
164
179
|
|
165
180
|
@typing.overload
|
166
|
-
def
|
181
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
167
182
|
"""
|
168
|
-
Specifies
|
169
|
-
|
170
|
-
This decorator is useful if this step may hang indefinitely.
|
183
|
+
Specifies the number of times the task corresponding
|
184
|
+
to a step needs to be retried.
|
171
185
|
|
172
|
-
This
|
173
|
-
|
174
|
-
|
186
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
187
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
188
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
175
189
|
|
176
|
-
|
177
|
-
|
190
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
191
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
192
|
+
ensuring that the flow execution can continue.
|
178
193
|
"""
|
179
194
|
...
|
180
195
|
|
181
196
|
@typing.overload
|
182
|
-
def
|
197
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
183
198
|
...
|
184
199
|
|
185
200
|
@typing.overload
|
186
|
-
def
|
201
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
187
202
|
...
|
188
203
|
|
189
|
-
def
|
204
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
190
205
|
"""
|
191
|
-
Specifies
|
192
|
-
|
193
|
-
This decorator is useful if this step may hang indefinitely.
|
206
|
+
Specifies the number of times the task corresponding
|
207
|
+
to a step needs to be retried.
|
194
208
|
|
195
|
-
This
|
196
|
-
|
197
|
-
|
209
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
210
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
211
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
198
212
|
|
199
|
-
|
200
|
-
|
213
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
214
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
215
|
+
ensuring that the flow execution can continue.
|
201
216
|
"""
|
202
217
|
...
|
203
218
|
|
@@ -233,51 +248,50 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
233
248
|
...
|
234
249
|
|
235
250
|
@typing.overload
|
236
|
-
def
|
251
|
+
def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
237
252
|
"""
|
238
|
-
|
239
|
-
|
240
|
-
Use `@resources` to specify the resource requirements
|
241
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
253
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
242
254
|
|
243
|
-
|
244
|
-
```
|
245
|
-
python myflow.py run --with batch
|
246
|
-
```
|
247
|
-
or
|
248
|
-
```
|
249
|
-
python myflow.py run --with kubernetes
|
250
|
-
```
|
251
|
-
which executes the flow on the desired system using the
|
252
|
-
requirements specified in `@resources`.
|
255
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
253
256
|
"""
|
254
257
|
...
|
255
258
|
|
256
259
|
@typing.overload
|
257
|
-
def
|
260
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
258
261
|
...
|
259
262
|
|
260
263
|
@typing.overload
|
261
|
-
def
|
264
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
262
265
|
...
|
263
266
|
|
264
|
-
def
|
267
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
265
268
|
"""
|
266
|
-
|
267
|
-
|
268
|
-
Use `@resources` to specify the resource requirements
|
269
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
269
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
270
270
|
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
271
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
272
|
+
"""
|
273
|
+
...
|
274
|
+
|
275
|
+
@typing.overload
|
276
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
277
|
+
"""
|
278
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
279
|
+
the execution of a step.
|
280
|
+
"""
|
281
|
+
...
|
282
|
+
|
283
|
+
@typing.overload
|
284
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
285
|
+
...
|
286
|
+
|
287
|
+
@typing.overload
|
288
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
289
|
+
...
|
290
|
+
|
291
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
292
|
+
"""
|
293
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
294
|
+
the execution of a step.
|
281
295
|
"""
|
282
296
|
...
|
283
297
|
|
@@ -303,23 +317,51 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
303
317
|
...
|
304
318
|
|
305
319
|
@typing.overload
|
306
|
-
def
|
320
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
307
321
|
"""
|
308
|
-
Specifies
|
322
|
+
Specifies the resources needed when executing this step.
|
323
|
+
|
324
|
+
Use `@resources` to specify the resource requirements
|
325
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
326
|
+
|
327
|
+
You can choose the compute layer on the command line by executing e.g.
|
328
|
+
```
|
329
|
+
python myflow.py run --with batch
|
330
|
+
```
|
331
|
+
or
|
332
|
+
```
|
333
|
+
python myflow.py run --with kubernetes
|
334
|
+
```
|
335
|
+
which executes the flow on the desired system using the
|
336
|
+
requirements specified in `@resources`.
|
309
337
|
"""
|
310
338
|
...
|
311
339
|
|
312
340
|
@typing.overload
|
313
|
-
def
|
341
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
314
342
|
...
|
315
343
|
|
316
344
|
@typing.overload
|
317
|
-
def
|
345
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
318
346
|
...
|
319
347
|
|
320
|
-
def
|
348
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
321
349
|
"""
|
322
|
-
Specifies
|
350
|
+
Specifies the resources needed when executing this step.
|
351
|
+
|
352
|
+
Use `@resources` to specify the resource requirements
|
353
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
354
|
+
|
355
|
+
You can choose the compute layer on the command line by executing e.g.
|
356
|
+
```
|
357
|
+
python myflow.py run --with batch
|
358
|
+
```
|
359
|
+
or
|
360
|
+
```
|
361
|
+
python myflow.py run --with kubernetes
|
362
|
+
```
|
363
|
+
which executes the flow on the desired system using the
|
364
|
+
requirements specified in `@resources`.
|
323
365
|
"""
|
324
366
|
...
|
325
367
|
|
@@ -330,25 +372,41 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
330
372
|
...
|
331
373
|
|
332
374
|
@typing.overload
|
333
|
-
def
|
375
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
334
376
|
"""
|
335
|
-
Specifies
|
336
|
-
|
377
|
+
Specifies a timeout for your step.
|
378
|
+
|
379
|
+
This decorator is useful if this step may hang indefinitely.
|
380
|
+
|
381
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
382
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
383
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
384
|
+
|
385
|
+
Note that all the values specified in parameters are added together so if you specify
|
386
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
337
387
|
"""
|
338
388
|
...
|
339
389
|
|
340
390
|
@typing.overload
|
341
|
-
def
|
391
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
342
392
|
...
|
343
393
|
|
344
394
|
@typing.overload
|
345
|
-
def
|
395
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
346
396
|
...
|
347
397
|
|
348
|
-
def
|
398
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
349
399
|
"""
|
350
|
-
Specifies
|
351
|
-
|
400
|
+
Specifies a timeout for your step.
|
401
|
+
|
402
|
+
This decorator is useful if this step may hang indefinitely.
|
403
|
+
|
404
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
405
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
406
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
407
|
+
|
408
|
+
Note that all the values specified in parameters are added together so if you specify
|
409
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
352
410
|
"""
|
353
411
|
...
|
354
412
|
|
@@ -384,46 +442,26 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
384
442
|
...
|
385
443
|
|
386
444
|
@typing.overload
|
387
|
-
def
|
445
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
388
446
|
"""
|
389
|
-
Specifies the
|
390
|
-
to a step needs to be retried.
|
391
|
-
|
392
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
393
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
394
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
447
|
+
Specifies the Conda environment for the step.
|
395
448
|
|
396
|
-
|
397
|
-
|
398
|
-
|
449
|
+
Information in this decorator will augment any
|
450
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
451
|
+
you can use `@conda_base` to set packages required by all
|
452
|
+
steps and use `@conda` to specify step-specific overrides.
|
399
453
|
"""
|
400
454
|
...
|
401
455
|
|
402
456
|
@typing.overload
|
403
|
-
def
|
457
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
404
458
|
...
|
405
459
|
|
406
460
|
@typing.overload
|
407
|
-
def
|
408
|
-
...
|
409
|
-
|
410
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
411
|
-
"""
|
412
|
-
Specifies the number of times the task corresponding
|
413
|
-
to a step needs to be retried.
|
414
|
-
|
415
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
416
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
417
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
418
|
-
|
419
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
420
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
421
|
-
ensuring that the flow execution can continue.
|
422
|
-
"""
|
461
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
423
462
|
...
|
424
463
|
|
425
|
-
|
426
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
464
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
427
465
|
"""
|
428
466
|
Specifies the Conda environment for the step.
|
429
467
|
|
@@ -435,40 +473,128 @@ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, s
|
|
435
473
|
...
|
436
474
|
|
437
475
|
@typing.overload
|
438
|
-
def
|
476
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
477
|
+
"""
|
478
|
+
Specifies the times when the flow should be run when running on a
|
479
|
+
production scheduler.
|
480
|
+
"""
|
439
481
|
...
|
440
482
|
|
441
483
|
@typing.overload
|
442
|
-
def
|
484
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
443
485
|
...
|
444
486
|
|
445
|
-
def
|
487
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
446
488
|
"""
|
447
|
-
Specifies the
|
448
|
-
|
449
|
-
Information in this decorator will augment any
|
450
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
451
|
-
you can use `@conda_base` to set packages required by all
|
452
|
-
steps and use `@conda` to specify step-specific overrides.
|
489
|
+
Specifies the times when the flow should be run when running on a
|
490
|
+
production scheduler.
|
453
491
|
"""
|
454
492
|
...
|
455
493
|
|
456
494
|
@typing.overload
|
457
|
-
def
|
495
|
+
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
458
496
|
"""
|
459
|
-
|
460
|
-
|
497
|
+
Specifies the flow(s) that this flow depends on.
|
498
|
+
|
499
|
+
```
|
500
|
+
@trigger_on_finish(flow='FooFlow')
|
501
|
+
```
|
502
|
+
or
|
503
|
+
```
|
504
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
505
|
+
```
|
506
|
+
This decorator respects the @project decorator and triggers the flow
|
507
|
+
when upstream runs within the same namespace complete successfully
|
508
|
+
|
509
|
+
Additionally, you can specify project aware upstream flow dependencies
|
510
|
+
by specifying the fully qualified project_flow_name.
|
511
|
+
```
|
512
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
513
|
+
```
|
514
|
+
or
|
515
|
+
```
|
516
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
517
|
+
```
|
518
|
+
|
519
|
+
You can also specify just the project or project branch (other values will be
|
520
|
+
inferred from the current project or project branch):
|
521
|
+
```
|
522
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
523
|
+
```
|
524
|
+
|
525
|
+
Note that `branch` is typically one of:
|
526
|
+
- `prod`
|
527
|
+
- `user.bob`
|
528
|
+
- `test.my_experiment`
|
529
|
+
- `prod.staging`
|
461
530
|
"""
|
462
531
|
...
|
463
532
|
|
464
533
|
@typing.overload
|
465
|
-
def
|
534
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
466
535
|
...
|
467
536
|
|
468
|
-
def
|
537
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
469
538
|
"""
|
470
|
-
|
471
|
-
|
539
|
+
Specifies the flow(s) that this flow depends on.
|
540
|
+
|
541
|
+
```
|
542
|
+
@trigger_on_finish(flow='FooFlow')
|
543
|
+
```
|
544
|
+
or
|
545
|
+
```
|
546
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
547
|
+
```
|
548
|
+
This decorator respects the @project decorator and triggers the flow
|
549
|
+
when upstream runs within the same namespace complete successfully
|
550
|
+
|
551
|
+
Additionally, you can specify project aware upstream flow dependencies
|
552
|
+
by specifying the fully qualified project_flow_name.
|
553
|
+
```
|
554
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
555
|
+
```
|
556
|
+
or
|
557
|
+
```
|
558
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
559
|
+
```
|
560
|
+
|
561
|
+
You can also specify just the project or project branch (other values will be
|
562
|
+
inferred from the current project or project branch):
|
563
|
+
```
|
564
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
565
|
+
```
|
566
|
+
|
567
|
+
Note that `branch` is typically one of:
|
568
|
+
- `prod`
|
569
|
+
- `user.bob`
|
570
|
+
- `test.my_experiment`
|
571
|
+
- `prod.staging`
|
572
|
+
"""
|
573
|
+
...
|
574
|
+
|
575
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
576
|
+
"""
|
577
|
+
Specifies what flows belong to the same project.
|
578
|
+
|
579
|
+
A project-specific namespace is created for all flows that
|
580
|
+
use the same `@project(name)`.
|
581
|
+
"""
|
582
|
+
...
|
583
|
+
|
584
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
585
|
+
"""
|
586
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
587
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
588
|
+
"""
|
589
|
+
...
|
590
|
+
|
591
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
592
|
+
"""
|
593
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
594
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
595
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
596
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
597
|
+
starts only after all sensors finish.
|
472
598
|
"""
|
473
599
|
...
|
474
600
|
|
@@ -545,22 +671,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
545
671
|
"""
|
546
672
|
...
|
547
673
|
|
548
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
549
|
-
"""
|
550
|
-
Specifies what flows belong to the same project.
|
551
|
-
|
552
|
-
A project-specific namespace is created for all flows that
|
553
|
-
use the same `@project(name)`.
|
554
|
-
"""
|
555
|
-
...
|
556
|
-
|
557
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
558
|
-
"""
|
559
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
560
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
561
|
-
"""
|
562
|
-
...
|
563
|
-
|
564
674
|
@typing.overload
|
565
675
|
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
566
676
|
"""
|
@@ -584,16 +694,6 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
|
|
584
694
|
"""
|
585
695
|
...
|
586
696
|
|
587
|
-
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
588
|
-
"""
|
589
|
-
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
590
|
-
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
591
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
592
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
593
|
-
starts only after all sensors finish.
|
594
|
-
"""
|
595
|
-
...
|
596
|
-
|
597
697
|
@typing.overload
|
598
698
|
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
599
699
|
"""
|
@@ -617,103 +717,3 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
|
|
617
717
|
"""
|
618
718
|
...
|
619
719
|
|
620
|
-
@typing.overload
|
621
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
622
|
-
"""
|
623
|
-
Specifies the times when the flow should be run when running on a
|
624
|
-
production scheduler.
|
625
|
-
"""
|
626
|
-
...
|
627
|
-
|
628
|
-
@typing.overload
|
629
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
630
|
-
...
|
631
|
-
|
632
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
633
|
-
"""
|
634
|
-
Specifies the times when the flow should be run when running on a
|
635
|
-
production scheduler.
|
636
|
-
"""
|
637
|
-
...
|
638
|
-
|
639
|
-
@typing.overload
|
640
|
-
def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
641
|
-
"""
|
642
|
-
Specifies the flow(s) that this flow depends on.
|
643
|
-
|
644
|
-
```
|
645
|
-
@trigger_on_finish(flow='FooFlow')
|
646
|
-
```
|
647
|
-
or
|
648
|
-
```
|
649
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
650
|
-
```
|
651
|
-
This decorator respects the @project decorator and triggers the flow
|
652
|
-
when upstream runs within the same namespace complete successfully
|
653
|
-
|
654
|
-
Additionally, you can specify project aware upstream flow dependencies
|
655
|
-
by specifying the fully qualified project_flow_name.
|
656
|
-
```
|
657
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
658
|
-
```
|
659
|
-
or
|
660
|
-
```
|
661
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
662
|
-
```
|
663
|
-
|
664
|
-
You can also specify just the project or project branch (other values will be
|
665
|
-
inferred from the current project or project branch):
|
666
|
-
```
|
667
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
668
|
-
```
|
669
|
-
|
670
|
-
Note that `branch` is typically one of:
|
671
|
-
- `prod`
|
672
|
-
- `user.bob`
|
673
|
-
- `test.my_experiment`
|
674
|
-
- `prod.staging`
|
675
|
-
"""
|
676
|
-
...
|
677
|
-
|
678
|
-
@typing.overload
|
679
|
-
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
680
|
-
...
|
681
|
-
|
682
|
-
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
683
|
-
"""
|
684
|
-
Specifies the flow(s) that this flow depends on.
|
685
|
-
|
686
|
-
```
|
687
|
-
@trigger_on_finish(flow='FooFlow')
|
688
|
-
```
|
689
|
-
or
|
690
|
-
```
|
691
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
692
|
-
```
|
693
|
-
This decorator respects the @project decorator and triggers the flow
|
694
|
-
when upstream runs within the same namespace complete successfully
|
695
|
-
|
696
|
-
Additionally, you can specify project aware upstream flow dependencies
|
697
|
-
by specifying the fully qualified project_flow_name.
|
698
|
-
```
|
699
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
700
|
-
```
|
701
|
-
or
|
702
|
-
```
|
703
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
704
|
-
```
|
705
|
-
|
706
|
-
You can also specify just the project or project branch (other values will be
|
707
|
-
inferred from the current project or project branch):
|
708
|
-
```
|
709
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
710
|
-
```
|
711
|
-
|
712
|
-
Note that `branch` is typically one of:
|
713
|
-
- `prod`
|
714
|
-
- `user.bob`
|
715
|
-
- `test.my_experiment`
|
716
|
-
- `prod.staging`
|
717
|
-
"""
|
718
|
-
...
|
719
|
-
|