metaflow-stubs 2.15.18__py2.py3-none-any.whl → 2.15.19__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of metaflow-stubs might be problematic. Click here for more details.

Files changed (155) hide show
  1. metaflow-stubs/__init__.pyi +472 -468
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +18 -18
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/multicore_utils.pyi +2 -2
  23. metaflow-stubs/parameters.pyi +4 -4
  24. metaflow-stubs/plugins/__init__.pyi +17 -15
  25. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  31. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  32. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  33. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  35. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  36. metaflow-stubs/plugins/argo/argo_workflows.pyi +6 -33
  37. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  38. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  39. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  40. metaflow-stubs/plugins/argo/exit_hooks.pyi +48 -0
  41. metaflow-stubs/plugins/aws/__init__.pyi +3 -3
  42. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  43. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  46. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  47. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  48. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  49. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  50. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +5 -5
  58. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  59. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  61. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  62. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  63. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  64. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  65. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  66. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_creator.pyi +4 -3
  69. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_decorator.pyi +13 -2
  71. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  75. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +10 -2
  78. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  79. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  80. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  81. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  82. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  83. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  84. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  85. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  86. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  87. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  88. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  89. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  90. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  91. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  92. metaflow-stubs/plugins/exit_hook/__init__.pyi +11 -0
  93. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +20 -0
  94. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  96. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  97. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  98. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  99. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  101. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  102. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  104. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  106. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  107. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  108. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  110. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  111. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  112. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  113. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  114. metaflow-stubs/plugins/pypi/conda_environment.pyi +6 -6
  115. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  116. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  119. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  121. metaflow-stubs/plugins/secrets/__init__.pyi +7 -3
  122. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  123. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +9 -49
  124. metaflow-stubs/plugins/secrets/secrets_func.pyi +32 -0
  125. metaflow-stubs/plugins/secrets/secrets_spec.pyi +42 -0
  126. metaflow-stubs/plugins/secrets/utils.pyi +28 -0
  127. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  128. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  129. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  130. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  131. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  132. metaflow-stubs/pylint_wrapper.pyi +2 -2
  133. metaflow-stubs/runner/__init__.pyi +2 -2
  134. metaflow-stubs/runner/deployer.pyi +30 -30
  135. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  136. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  137. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  138. metaflow-stubs/runner/nbrun.pyi +2 -2
  139. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  140. metaflow-stubs/runner/utils.pyi +4 -4
  141. metaflow-stubs/system/__init__.pyi +2 -2
  142. metaflow-stubs/system/system_logger.pyi +3 -3
  143. metaflow-stubs/system/system_monitor.pyi +2 -2
  144. metaflow-stubs/tagging_util.pyi +2 -2
  145. metaflow-stubs/tuple_util.pyi +2 -2
  146. metaflow-stubs/user_configs/__init__.pyi +2 -2
  147. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  148. metaflow-stubs/user_configs/config_options.pyi +4 -4
  149. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  150. metaflow-stubs/version.pyi +2 -2
  151. {metaflow_stubs-2.15.18.dist-info → metaflow_stubs-2.15.19.dist-info}/METADATA +2 -2
  152. metaflow_stubs-2.15.19.dist-info/RECORD +155 -0
  153. metaflow_stubs-2.15.18.dist-info/RECORD +0 -149
  154. {metaflow_stubs-2.15.18.dist-info → metaflow_stubs-2.15.19.dist-info}/WHEEL +0 -0
  155. {metaflow_stubs-2.15.18.dist-info → metaflow_stubs-2.15.19.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.18 #
4
- # Generated on 2025-06-19T23:01:30.833215 #
3
+ # MF version: 2.15.19 #
4
+ # Generated on 2025-07-10T01:18:40.943007 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -36,8 +36,8 @@ from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
38
  from . import metaflow_git as metaflow_git
39
- from . import events as events
40
39
  from . import tuple_util as tuple_util
40
+ from . import events as events
41
41
  from . import runner as runner
42
42
  from . import plugins as plugins
43
43
  from .plugins.datatools.s3.s3 import S3 as S3
@@ -147,90 +147,21 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
147
147
  ...
148
148
 
149
149
  @typing.overload
150
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
151
- """
152
- Specifies environment variables to be set prior to the execution of a step.
153
-
154
-
155
- Parameters
156
- ----------
157
- vars : Dict[str, str], default {}
158
- Dictionary of environment variables to set.
159
- """
160
- ...
161
-
162
- @typing.overload
163
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
164
- ...
165
-
166
- @typing.overload
167
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
168
- ...
169
-
170
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
171
- """
172
- Specifies environment variables to be set prior to the execution of a step.
173
-
174
-
175
- Parameters
176
- ----------
177
- vars : Dict[str, str], default {}
178
- Dictionary of environment variables to set.
179
- """
180
- ...
181
-
182
- @typing.overload
183
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
150
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
184
151
  """
185
- Specifies the number of times the task corresponding
186
- to a step needs to be retried.
187
-
188
- This decorator is useful for handling transient errors, such as networking issues.
189
- If your task contains operations that can't be retried safely, e.g. database updates,
190
- it is advisable to annotate it with `@retry(times=0)`.
191
-
192
- This can be used in conjunction with the `@catch` decorator. The `@catch`
193
- decorator will execute a no-op task after all retries have been exhausted,
194
- ensuring that the flow execution can continue.
195
-
196
-
197
- Parameters
198
- ----------
199
- times : int, default 3
200
- Number of times to retry this task.
201
- minutes_between_retries : int, default 2
202
- Number of minutes between retries.
152
+ Decorator prototype for all step decorators. This function gets specialized
153
+ and imported for all decorators types by _import_plugin_decorators().
203
154
  """
204
155
  ...
205
156
 
206
157
  @typing.overload
207
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
208
- ...
209
-
210
- @typing.overload
211
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
158
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
212
159
  ...
213
160
 
214
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
161
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
215
162
  """
216
- Specifies the number of times the task corresponding
217
- to a step needs to be retried.
218
-
219
- This decorator is useful for handling transient errors, such as networking issues.
220
- If your task contains operations that can't be retried safely, e.g. database updates,
221
- it is advisable to annotate it with `@retry(times=0)`.
222
-
223
- This can be used in conjunction with the `@catch` decorator. The `@catch`
224
- decorator will execute a no-op task after all retries have been exhausted,
225
- ensuring that the flow execution can continue.
226
-
227
-
228
- Parameters
229
- ----------
230
- times : int, default 3
231
- Number of times to retry this task.
232
- minutes_between_retries : int, default 2
233
- Number of minutes between retries.
163
+ Decorator prototype for all step decorators. This function gets specialized
164
+ and imported for all decorators types by _import_plugin_decorators().
234
165
  """
235
166
  ...
236
167
 
@@ -286,53 +217,81 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
286
217
  ...
287
218
 
288
219
  @typing.overload
289
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
220
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
290
221
  """
291
- Specifies that the step will success under all circumstances.
222
+ Specifies the resources needed when executing this step.
292
223
 
293
- The decorator will create an optional artifact, specified by `var`, which
294
- contains the exception raised. You can use it to detect the presence
295
- of errors, indicating that all happy-path artifacts produced by the step
296
- are missing.
224
+ Use `@resources` to specify the resource requirements
225
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
226
+
227
+ You can choose the compute layer on the command line by executing e.g.
228
+ ```
229
+ python myflow.py run --with batch
230
+ ```
231
+ or
232
+ ```
233
+ python myflow.py run --with kubernetes
234
+ ```
235
+ which executes the flow on the desired system using the
236
+ requirements specified in `@resources`.
297
237
 
298
238
 
299
239
  Parameters
300
240
  ----------
301
- var : str, optional, default None
302
- Name of the artifact in which to store the caught exception.
303
- If not specified, the exception is not stored.
304
- print_exception : bool, default True
305
- Determines whether or not the exception is printed to
306
- stdout when caught.
241
+ cpu : int, default 1
242
+ Number of CPUs required for this step.
243
+ gpu : int, optional, default None
244
+ Number of GPUs required for this step.
245
+ disk : int, optional, default None
246
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
247
+ memory : int, default 4096
248
+ Memory size (in MB) required for this step.
249
+ shared_memory : int, optional, default None
250
+ The value for the size (in MiB) of the /dev/shm volume for this step.
251
+ This parameter maps to the `--shm-size` option in Docker.
307
252
  """
308
253
  ...
309
254
 
310
255
  @typing.overload
311
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
256
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
312
257
  ...
313
258
 
314
259
  @typing.overload
315
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
260
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
316
261
  ...
317
262
 
318
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
263
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
319
264
  """
320
- Specifies that the step will success under all circumstances.
265
+ Specifies the resources needed when executing this step.
321
266
 
322
- The decorator will create an optional artifact, specified by `var`, which
323
- contains the exception raised. You can use it to detect the presence
324
- of errors, indicating that all happy-path artifacts produced by the step
325
- are missing.
267
+ Use `@resources` to specify the resource requirements
268
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
269
+
270
+ You can choose the compute layer on the command line by executing e.g.
271
+ ```
272
+ python myflow.py run --with batch
273
+ ```
274
+ or
275
+ ```
276
+ python myflow.py run --with kubernetes
277
+ ```
278
+ which executes the flow on the desired system using the
279
+ requirements specified in `@resources`.
326
280
 
327
281
 
328
282
  Parameters
329
283
  ----------
330
- var : str, optional, default None
331
- Name of the artifact in which to store the caught exception.
332
- If not specified, the exception is not stored.
333
- print_exception : bool, default True
334
- Determines whether or not the exception is printed to
335
- stdout when caught.
284
+ cpu : int, default 1
285
+ Number of CPUs required for this step.
286
+ gpu : int, optional, default None
287
+ Number of GPUs required for this step.
288
+ disk : int, optional, default None
289
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
290
+ memory : int, default 4096
291
+ Memory size (in MB) required for this step.
292
+ shared_memory : int, optional, default None
293
+ The value for the size (in MiB) of the /dev/shm volume for this step.
294
+ This parameter maps to the `--shm-size` option in Docker.
336
295
  """
337
296
  ...
338
297
 
@@ -426,140 +385,90 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
426
385
  ...
427
386
 
428
387
  @typing.overload
429
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
388
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
430
389
  """
431
- Specifies a timeout for your step.
432
-
433
- This decorator is useful if this step may hang indefinitely.
390
+ Specifies the number of times the task corresponding
391
+ to a step needs to be retried.
434
392
 
435
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
436
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
437
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
393
+ This decorator is useful for handling transient errors, such as networking issues.
394
+ If your task contains operations that can't be retried safely, e.g. database updates,
395
+ it is advisable to annotate it with `@retry(times=0)`.
438
396
 
439
- Note that all the values specified in parameters are added together so if you specify
440
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
397
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
398
+ decorator will execute a no-op task after all retries have been exhausted,
399
+ ensuring that the flow execution can continue.
441
400
 
442
401
 
443
402
  Parameters
444
403
  ----------
445
- seconds : int, default 0
446
- Number of seconds to wait prior to timing out.
447
- minutes : int, default 0
448
- Number of minutes to wait prior to timing out.
449
- hours : int, default 0
450
- Number of hours to wait prior to timing out.
404
+ times : int, default 3
405
+ Number of times to retry this task.
406
+ minutes_between_retries : int, default 2
407
+ Number of minutes between retries.
451
408
  """
452
409
  ...
453
410
 
454
411
  @typing.overload
455
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
412
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
456
413
  ...
457
414
 
458
415
  @typing.overload
459
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
416
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
460
417
  ...
461
418
 
462
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
419
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
463
420
  """
464
- Specifies a timeout for your step.
465
-
466
- This decorator is useful if this step may hang indefinitely.
421
+ Specifies the number of times the task corresponding
422
+ to a step needs to be retried.
467
423
 
468
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
469
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
470
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
424
+ This decorator is useful for handling transient errors, such as networking issues.
425
+ If your task contains operations that can't be retried safely, e.g. database updates,
426
+ it is advisable to annotate it with `@retry(times=0)`.
471
427
 
472
- Note that all the values specified in parameters are added together so if you specify
473
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
428
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
429
+ decorator will execute a no-op task after all retries have been exhausted,
430
+ ensuring that the flow execution can continue.
474
431
 
475
432
 
476
433
  Parameters
477
434
  ----------
478
- seconds : int, default 0
479
- Number of seconds to wait prior to timing out.
480
- minutes : int, default 0
481
- Number of minutes to wait prior to timing out.
482
- hours : int, default 0
483
- Number of hours to wait prior to timing out.
435
+ times : int, default 3
436
+ Number of times to retry this task.
437
+ minutes_between_retries : int, default 2
438
+ Number of minutes between retries.
484
439
  """
485
440
  ...
486
441
 
487
442
  @typing.overload
488
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
443
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
489
444
  """
490
- Specifies the resources needed when executing this step.
491
-
492
- Use `@resources` to specify the resource requirements
493
- independently of the specific compute layer (`@batch`, `@kubernetes`).
494
-
495
- You can choose the compute layer on the command line by executing e.g.
496
- ```
497
- python myflow.py run --with batch
498
- ```
499
- or
500
- ```
501
- python myflow.py run --with kubernetes
502
- ```
503
- which executes the flow on the desired system using the
504
- requirements specified in `@resources`.
445
+ Specifies environment variables to be set prior to the execution of a step.
505
446
 
506
447
 
507
448
  Parameters
508
449
  ----------
509
- cpu : int, default 1
510
- Number of CPUs required for this step.
511
- gpu : int, optional, default None
512
- Number of GPUs required for this step.
513
- disk : int, optional, default None
514
- Disk size (in MB) required for this step. Only applies on Kubernetes.
515
- memory : int, default 4096
516
- Memory size (in MB) required for this step.
517
- shared_memory : int, optional, default None
518
- The value for the size (in MiB) of the /dev/shm volume for this step.
519
- This parameter maps to the `--shm-size` option in Docker.
450
+ vars : Dict[str, str], default {}
451
+ Dictionary of environment variables to set.
520
452
  """
521
453
  ...
522
454
 
523
455
  @typing.overload
524
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
456
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
525
457
  ...
526
458
 
527
459
  @typing.overload
528
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
460
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
529
461
  ...
530
462
 
531
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
463
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
532
464
  """
533
- Specifies the resources needed when executing this step.
534
-
535
- Use `@resources` to specify the resource requirements
536
- independently of the specific compute layer (`@batch`, `@kubernetes`).
537
-
538
- You can choose the compute layer on the command line by executing e.g.
539
- ```
540
- python myflow.py run --with batch
541
- ```
542
- or
543
- ```
544
- python myflow.py run --with kubernetes
545
- ```
546
- which executes the flow on the desired system using the
547
- requirements specified in `@resources`.
465
+ Specifies environment variables to be set prior to the execution of a step.
548
466
 
549
467
 
550
468
  Parameters
551
469
  ----------
552
- cpu : int, default 1
553
- Number of CPUs required for this step.
554
- gpu : int, optional, default None
555
- Number of GPUs required for this step.
556
- disk : int, optional, default None
557
- Disk size (in MB) required for this step. Only applies on Kubernetes.
558
- memory : int, default 4096
559
- Memory size (in MB) required for this step.
560
- shared_memory : int, optional, default None
561
- The value for the size (in MiB) of the /dev/shm volume for this step.
562
- This parameter maps to the `--shm-size` option in Docker.
470
+ vars : Dict[str, str], default {}
471
+ Dictionary of environment variables to set.
563
472
  """
564
473
  ...
565
474
 
@@ -712,60 +621,6 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
712
621
  """
713
622
  ...
714
623
 
715
- @typing.overload
716
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
717
- """
718
- Decorator prototype for all step decorators. This function gets specialized
719
- and imported for all decorators types by _import_plugin_decorators().
720
- """
721
- ...
722
-
723
- @typing.overload
724
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
725
- ...
726
-
727
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
728
- """
729
- Decorator prototype for all step decorators. This function gets specialized
730
- and imported for all decorators types by _import_plugin_decorators().
731
- """
732
- ...
733
-
734
- @typing.overload
735
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
736
- """
737
- Specifies secrets to be retrieved and injected as environment variables prior to
738
- the execution of a step.
739
-
740
-
741
- Parameters
742
- ----------
743
- sources : List[Union[str, Dict[str, Any]]], default: []
744
- List of secret specs, defining how the secrets are to be retrieved
745
- """
746
- ...
747
-
748
- @typing.overload
749
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
750
- ...
751
-
752
- @typing.overload
753
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
754
- ...
755
-
756
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
757
- """
758
- Specifies secrets to be retrieved and injected as environment variables prior to
759
- the execution of a step.
760
-
761
-
762
- Parameters
763
- ----------
764
- sources : List[Union[str, Dict[str, Any]]], default: []
765
- List of secret specs, defining how the secrets are to be retrieved
766
- """
767
- ...
768
-
769
624
  @typing.overload
770
625
  def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
771
626
  """
@@ -825,6 +680,57 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
825
680
  """
826
681
  ...
827
682
 
683
+ @typing.overload
684
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
685
+ """
686
+ Specifies that the step will success under all circumstances.
687
+
688
+ The decorator will create an optional artifact, specified by `var`, which
689
+ contains the exception raised. You can use it to detect the presence
690
+ of errors, indicating that all happy-path artifacts produced by the step
691
+ are missing.
692
+
693
+
694
+ Parameters
695
+ ----------
696
+ var : str, optional, default None
697
+ Name of the artifact in which to store the caught exception.
698
+ If not specified, the exception is not stored.
699
+ print_exception : bool, default True
700
+ Determines whether or not the exception is printed to
701
+ stdout when caught.
702
+ """
703
+ ...
704
+
705
+ @typing.overload
706
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
707
+ ...
708
+
709
+ @typing.overload
710
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
711
+ ...
712
+
713
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
714
+ """
715
+ Specifies that the step will success under all circumstances.
716
+
717
+ The decorator will create an optional artifact, specified by `var`, which
718
+ contains the exception raised. You can use it to detect the presence
719
+ of errors, indicating that all happy-path artifacts produced by the step
720
+ are missing.
721
+
722
+
723
+ Parameters
724
+ ----------
725
+ var : str, optional, default None
726
+ Name of the artifact in which to store the caught exception.
727
+ If not specified, the exception is not stored.
728
+ print_exception : bool, default True
729
+ Determines whether or not the exception is printed to
730
+ stdout when caught.
731
+ """
732
+ ...
733
+
828
734
  @typing.overload
829
735
  def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
830
736
  """
@@ -875,146 +781,227 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
875
781
  ...
876
782
 
877
783
  @typing.overload
878
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
784
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
879
785
  """
880
- Specifies the Conda environment for all steps of the flow.
786
+ Specifies a timeout for your step.
881
787
 
882
- Use `@conda_base` to set common libraries required by all
883
- steps and use `@conda` to specify step-specific additions.
788
+ This decorator is useful if this step may hang indefinitely.
789
+
790
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
791
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
792
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
793
+
794
+ Note that all the values specified in parameters are added together so if you specify
795
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
884
796
 
885
797
 
886
798
  Parameters
887
799
  ----------
888
- packages : Dict[str, str], default {}
889
- Packages to use for this flow. The key is the name of the package
890
- and the value is the version to use.
891
- libraries : Dict[str, str], default {}
892
- Supported for backward compatibility. When used with packages, packages will take precedence.
893
- python : str, optional, default None
894
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
895
- that the version used will correspond to the version of the Python interpreter used to start the run.
896
- disabled : bool, default False
897
- If set to True, disables Conda.
800
+ seconds : int, default 0
801
+ Number of seconds to wait prior to timing out.
802
+ minutes : int, default 0
803
+ Number of minutes to wait prior to timing out.
804
+ hours : int, default 0
805
+ Number of hours to wait prior to timing out.
898
806
  """
899
807
  ...
900
808
 
901
809
  @typing.overload
902
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
810
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
903
811
  ...
904
812
 
905
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
813
+ @typing.overload
814
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
815
+ ...
816
+
817
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
906
818
  """
907
- Specifies the Conda environment for all steps of the flow.
819
+ Specifies a timeout for your step.
908
820
 
909
- Use `@conda_base` to set common libraries required by all
910
- steps and use `@conda` to specify step-specific additions.
821
+ This decorator is useful if this step may hang indefinitely.
822
+
823
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
824
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
825
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
826
+
827
+ Note that all the values specified in parameters are added together so if you specify
828
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
911
829
 
912
830
 
913
831
  Parameters
914
832
  ----------
915
- packages : Dict[str, str], default {}
916
- Packages to use for this flow. The key is the name of the package
917
- and the value is the version to use.
918
- libraries : Dict[str, str], default {}
919
- Supported for backward compatibility. When used with packages, packages will take precedence.
920
- python : str, optional, default None
921
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
922
- that the version used will correspond to the version of the Python interpreter used to start the run.
923
- disabled : bool, default False
924
- If set to True, disables Conda.
833
+ seconds : int, default 0
834
+ Number of seconds to wait prior to timing out.
835
+ minutes : int, default 0
836
+ Number of minutes to wait prior to timing out.
837
+ hours : int, default 0
838
+ Number of hours to wait prior to timing out.
925
839
  """
926
840
  ...
927
841
 
928
842
  @typing.overload
929
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
843
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
930
844
  """
931
- Specifies the event(s) that this flow depends on.
932
-
933
- ```
934
- @trigger(event='foo')
935
- ```
936
- or
937
- ```
938
- @trigger(events=['foo', 'bar'])
939
- ```
940
-
941
- Additionally, you can specify the parameter mappings
942
- to map event payload to Metaflow parameters for the flow.
943
- ```
944
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
945
- ```
946
- or
947
- ```
948
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
949
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
950
- ```
951
-
952
- 'parameters' can also be a list of strings and tuples like so:
953
- ```
954
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
955
- ```
956
- This is equivalent to:
957
- ```
958
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
959
- ```
845
+ Specifies secrets to be retrieved and injected as environment variables prior to
846
+ the execution of a step.
960
847
 
961
848
 
962
849
  Parameters
963
850
  ----------
964
- event : Union[str, Dict[str, Any]], optional, default None
965
- Event dependency for this flow.
966
- events : List[Union[str, Dict[str, Any]]], default []
967
- Events dependency for this flow.
968
- options : Dict[str, Any], default {}
969
- Backend-specific configuration for tuning eventing behavior.
851
+ sources : List[Union[str, Dict[str, Any]]], default: []
852
+ List of secret specs, defining how the secrets are to be retrieved
853
+ role : str, optional, default: None
854
+ Role to use for fetching secrets
970
855
  """
971
856
  ...
972
857
 
973
858
  @typing.overload
974
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
859
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
975
860
  ...
976
861
 
977
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
862
+ @typing.overload
863
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
864
+ ...
865
+
866
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
978
867
  """
979
- Specifies the event(s) that this flow depends on.
868
+ Specifies secrets to be retrieved and injected as environment variables prior to
869
+ the execution of a step.
980
870
 
981
- ```
982
- @trigger(event='foo')
983
- ```
984
- or
985
- ```
986
- @trigger(events=['foo', 'bar'])
987
- ```
988
871
 
989
- Additionally, you can specify the parameter mappings
990
- to map event payload to Metaflow parameters for the flow.
991
- ```
992
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
993
- ```
994
- or
995
- ```
996
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
997
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
998
- ```
872
+ Parameters
873
+ ----------
874
+ sources : List[Union[str, Dict[str, Any]]], default: []
875
+ List of secret specs, defining how the secrets are to be retrieved
876
+ role : str, optional, default: None
877
+ Role to use for fetching secrets
878
+ """
879
+ ...
880
+
881
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
882
+ """
883
+ Specifies what flows belong to the same project.
999
884
 
1000
- 'parameters' can also be a list of strings and tuples like so:
1001
- ```
1002
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1003
- ```
1004
- This is equivalent to:
1005
- ```
1006
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1007
- ```
885
+ A project-specific namespace is created for all flows that
886
+ use the same `@project(name)`.
1008
887
 
1009
888
 
1010
889
  Parameters
1011
890
  ----------
1012
- event : Union[str, Dict[str, Any]], optional, default None
1013
- Event dependency for this flow.
1014
- events : List[Union[str, Dict[str, Any]]], default []
1015
- Events dependency for this flow.
1016
- options : Dict[str, Any], default {}
1017
- Backend-specific configuration for tuning eventing behavior.
891
+ name : str
892
+ Project name. Make sure that the name is unique amongst all
893
+ projects that use the same production scheduler. The name may
894
+ contain only lowercase alphanumeric characters and underscores.
895
+
896
+ branch : Optional[str], default None
897
+ The branch to use. If not specified, the branch is set to
898
+ `user.<username>` unless `production` is set to `True`. This can
899
+ also be set on the command line using `--branch` as a top-level option.
900
+ It is an error to specify `branch` in the decorator and on the command line.
901
+
902
+ production : bool, default False
903
+ Whether or not the branch is the production branch. This can also be set on the
904
+ command line using `--production` as a top-level option. It is an error to specify
905
+ `production` in the decorator and on the command line.
906
+ The project branch name will be:
907
+ - if `branch` is specified:
908
+ - if `production` is True: `prod.<branch>`
909
+ - if `production` is False: `test.<branch>`
910
+ - if `branch` is not specified:
911
+ - if `production` is True: `prod`
912
+ - if `production` is False: `user.<username>`
913
+ """
914
+ ...
915
+
916
+ @typing.overload
917
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
918
+ """
919
+ Specifies the times when the flow should be run when running on a
920
+ production scheduler.
921
+
922
+
923
+ Parameters
924
+ ----------
925
+ hourly : bool, default False
926
+ Run the workflow hourly.
927
+ daily : bool, default True
928
+ Run the workflow daily.
929
+ weekly : bool, default False
930
+ Run the workflow weekly.
931
+ cron : str, optional, default None
932
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
933
+ specified by this expression.
934
+ timezone : str, optional, default None
935
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
936
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
937
+ """
938
+ ...
939
+
940
+ @typing.overload
941
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
942
+ ...
943
+
944
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
945
+ """
946
+ Specifies the times when the flow should be run when running on a
947
+ production scheduler.
948
+
949
+
950
+ Parameters
951
+ ----------
952
+ hourly : bool, default False
953
+ Run the workflow hourly.
954
+ daily : bool, default True
955
+ Run the workflow daily.
956
+ weekly : bool, default False
957
+ Run the workflow weekly.
958
+ cron : str, optional, default None
959
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
960
+ specified by this expression.
961
+ timezone : str, optional, default None
962
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
963
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
964
+ """
965
+ ...
966
+
967
+ @typing.overload
968
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
969
+ """
970
+ Specifies the PyPI packages for all steps of the flow.
971
+
972
+ Use `@pypi_base` to set common packages required by all
973
+ steps and use `@pypi` to specify step-specific overrides.
974
+
975
+ Parameters
976
+ ----------
977
+ packages : Dict[str, str], default: {}
978
+ Packages to use for this flow. The key is the name of the package
979
+ and the value is the version to use.
980
+ python : str, optional, default: None
981
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
982
+ that the version used will correspond to the version of the Python interpreter used to start the run.
983
+ """
984
+ ...
985
+
986
+ @typing.overload
987
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
988
+ ...
989
+
990
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
991
+ """
992
+ Specifies the PyPI packages for all steps of the flow.
993
+
994
+ Use `@pypi_base` to set common packages required by all
995
+ steps and use `@pypi` to specify step-specific overrides.
996
+
997
+ Parameters
998
+ ----------
999
+ packages : Dict[str, str], default: {}
1000
+ Packages to use for this flow. The key is the name of the package
1001
+ and the value is the version to use.
1002
+ python : str, optional, default: None
1003
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1004
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1018
1005
  """
1019
1006
  ...
1020
1007
 
@@ -1119,137 +1106,157 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1119
1106
  """
1120
1107
  ...
1121
1108
 
1122
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1109
+ @typing.overload
1110
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1123
1111
  """
1124
- Specifies what flows belong to the same project.
1125
-
1126
- A project-specific namespace is created for all flows that
1127
- use the same `@project(name)`.
1128
-
1112
+ Specifies the event(s) that this flow depends on.
1129
1113
 
1130
- Parameters
1131
- ----------
1132
- name : str
1133
- Project name. Make sure that the name is unique amongst all
1134
- projects that use the same production scheduler. The name may
1135
- contain only lowercase alphanumeric characters and underscores.
1114
+ ```
1115
+ @trigger(event='foo')
1116
+ ```
1117
+ or
1118
+ ```
1119
+ @trigger(events=['foo', 'bar'])
1120
+ ```
1136
1121
 
1137
- branch : Optional[str], default None
1138
- The branch to use. If not specified, the branch is set to
1139
- `user.<username>` unless `production` is set to `True`. This can
1140
- also be set on the command line using `--branch` as a top-level option.
1141
- It is an error to specify `branch` in the decorator and on the command line.
1122
+ Additionally, you can specify the parameter mappings
1123
+ to map event payload to Metaflow parameters for the flow.
1124
+ ```
1125
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1126
+ ```
1127
+ or
1128
+ ```
1129
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1130
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1131
+ ```
1142
1132
 
1143
- production : bool, default False
1144
- Whether or not the branch is the production branch. This can also be set on the
1145
- command line using `--production` as a top-level option. It is an error to specify
1146
- `production` in the decorator and on the command line.
1147
- The project branch name will be:
1148
- - if `branch` is specified:
1149
- - if `production` is True: `prod.<branch>`
1150
- - if `production` is False: `test.<branch>`
1151
- - if `branch` is not specified:
1152
- - if `production` is True: `prod`
1153
- - if `production` is False: `user.<username>`
1154
- """
1155
- ...
1156
-
1157
- @typing.overload
1158
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1159
- """
1160
- Specifies the PyPI packages for all steps of the flow.
1133
+ 'parameters' can also be a list of strings and tuples like so:
1134
+ ```
1135
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1136
+ ```
1137
+ This is equivalent to:
1138
+ ```
1139
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1140
+ ```
1161
1141
 
1162
- Use `@pypi_base` to set common packages required by all
1163
- steps and use `@pypi` to specify step-specific overrides.
1164
1142
 
1165
1143
  Parameters
1166
1144
  ----------
1167
- packages : Dict[str, str], default: {}
1168
- Packages to use for this flow. The key is the name of the package
1169
- and the value is the version to use.
1170
- python : str, optional, default: None
1171
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1172
- that the version used will correspond to the version of the Python interpreter used to start the run.
1145
+ event : Union[str, Dict[str, Any]], optional, default None
1146
+ Event dependency for this flow.
1147
+ events : List[Union[str, Dict[str, Any]]], default []
1148
+ Events dependency for this flow.
1149
+ options : Dict[str, Any], default {}
1150
+ Backend-specific configuration for tuning eventing behavior.
1173
1151
  """
1174
1152
  ...
1175
1153
 
1176
1154
  @typing.overload
1177
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1155
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1178
1156
  ...
1179
1157
 
1180
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1158
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1181
1159
  """
1182
- Specifies the PyPI packages for all steps of the flow.
1160
+ Specifies the event(s) that this flow depends on.
1161
+
1162
+ ```
1163
+ @trigger(event='foo')
1164
+ ```
1165
+ or
1166
+ ```
1167
+ @trigger(events=['foo', 'bar'])
1168
+ ```
1169
+
1170
+ Additionally, you can specify the parameter mappings
1171
+ to map event payload to Metaflow parameters for the flow.
1172
+ ```
1173
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1174
+ ```
1175
+ or
1176
+ ```
1177
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1178
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1179
+ ```
1180
+
1181
+ 'parameters' can also be a list of strings and tuples like so:
1182
+ ```
1183
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1184
+ ```
1185
+ This is equivalent to:
1186
+ ```
1187
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1188
+ ```
1183
1189
 
1184
- Use `@pypi_base` to set common packages required by all
1185
- steps and use `@pypi` to specify step-specific overrides.
1186
1190
 
1187
1191
  Parameters
1188
1192
  ----------
1189
- packages : Dict[str, str], default: {}
1190
- Packages to use for this flow. The key is the name of the package
1191
- and the value is the version to use.
1192
- python : str, optional, default: None
1193
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1194
- that the version used will correspond to the version of the Python interpreter used to start the run.
1193
+ event : Union[str, Dict[str, Any]], optional, default None
1194
+ Event dependency for this flow.
1195
+ events : List[Union[str, Dict[str, Any]]], default []
1196
+ Events dependency for this flow.
1197
+ options : Dict[str, Any], default {}
1198
+ Backend-specific configuration for tuning eventing behavior.
1195
1199
  """
1196
1200
  ...
1197
1201
 
1198
1202
  @typing.overload
1199
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1203
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1200
1204
  """
1201
- Specifies the times when the flow should be run when running on a
1202
- production scheduler.
1205
+ Specifies the Conda environment for all steps of the flow.
1206
+
1207
+ Use `@conda_base` to set common libraries required by all
1208
+ steps and use `@conda` to specify step-specific additions.
1203
1209
 
1204
1210
 
1205
1211
  Parameters
1206
1212
  ----------
1207
- hourly : bool, default False
1208
- Run the workflow hourly.
1209
- daily : bool, default True
1210
- Run the workflow daily.
1211
- weekly : bool, default False
1212
- Run the workflow weekly.
1213
- cron : str, optional, default None
1214
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1215
- specified by this expression.
1216
- timezone : str, optional, default None
1217
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1218
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1213
+ packages : Dict[str, str], default {}
1214
+ Packages to use for this flow. The key is the name of the package
1215
+ and the value is the version to use.
1216
+ libraries : Dict[str, str], default {}
1217
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1218
+ python : str, optional, default None
1219
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1220
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1221
+ disabled : bool, default False
1222
+ If set to True, disables Conda.
1219
1223
  """
1220
1224
  ...
1221
1225
 
1222
1226
  @typing.overload
1223
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1227
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1224
1228
  ...
1225
1229
 
1226
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1230
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1227
1231
  """
1228
- Specifies the times when the flow should be run when running on a
1229
- production scheduler.
1232
+ Specifies the Conda environment for all steps of the flow.
1233
+
1234
+ Use `@conda_base` to set common libraries required by all
1235
+ steps and use `@conda` to specify step-specific additions.
1230
1236
 
1231
1237
 
1232
1238
  Parameters
1233
1239
  ----------
1234
- hourly : bool, default False
1235
- Run the workflow hourly.
1236
- daily : bool, default True
1237
- Run the workflow daily.
1238
- weekly : bool, default False
1239
- Run the workflow weekly.
1240
- cron : str, optional, default None
1241
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1242
- specified by this expression.
1243
- timezone : str, optional, default None
1244
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1245
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1240
+ packages : Dict[str, str], default {}
1241
+ Packages to use for this flow. The key is the name of the package
1242
+ and the value is the version to use.
1243
+ libraries : Dict[str, str], default {}
1244
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1245
+ python : str, optional, default None
1246
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1247
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1248
+ disabled : bool, default False
1249
+ If set to True, disables Conda.
1246
1250
  """
1247
1251
  ...
1248
1252
 
1249
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1253
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1250
1254
  """
1251
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1252
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1255
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1256
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1257
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1258
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1259
+ starts only after all sensors finish.
1253
1260
 
1254
1261
 
1255
1262
  Parameters
@@ -1271,31 +1278,25 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1271
1278
  Name of the sensor on Airflow
1272
1279
  description : str
1273
1280
  Description of sensor in the Airflow UI
1274
- external_dag_id : str
1275
- The dag_id that contains the task you want to wait for.
1276
- external_task_ids : List[str]
1277
- The list of task_ids that you want to wait for.
1278
- If None (default value) the sensor waits for the DAG. (Default: None)
1279
- allowed_states : List[str]
1280
- Iterable of allowed states, (Default: ['success'])
1281
- failed_states : List[str]
1282
- Iterable of failed or dis-allowed states. (Default: None)
1283
- execution_delta : datetime.timedelta
1284
- time difference with the previous execution to look at,
1285
- the default is the same logical date as the current task or DAG. (Default: None)
1286
- check_existence: bool
1287
- Set to True to check if the external task exists or check if
1288
- the DAG to wait for exists. (Default: True)
1281
+ bucket_key : Union[str, List[str]]
1282
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1283
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1284
+ bucket_name : str
1285
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1286
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1287
+ wildcard_match : bool
1288
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1289
+ aws_conn_id : str
1290
+ a reference to the s3 connection on Airflow. (Default: None)
1291
+ verify : bool
1292
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1289
1293
  """
1290
1294
  ...
1291
1295
 
1292
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1296
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1293
1297
  """
1294
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1295
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1296
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1297
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1298
- starts only after all sensors finish.
1298
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1299
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1299
1300
 
1300
1301
 
1301
1302
  Parameters
@@ -1317,18 +1318,21 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1317
1318
  Name of the sensor on Airflow
1318
1319
  description : str
1319
1320
  Description of sensor in the Airflow UI
1320
- bucket_key : Union[str, List[str]]
1321
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1322
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1323
- bucket_name : str
1324
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1325
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1326
- wildcard_match : bool
1327
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1328
- aws_conn_id : str
1329
- a reference to the s3 connection on Airflow. (Default: None)
1330
- verify : bool
1331
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1321
+ external_dag_id : str
1322
+ The dag_id that contains the task you want to wait for.
1323
+ external_task_ids : List[str]
1324
+ The list of task_ids that you want to wait for.
1325
+ If None (default value) the sensor waits for the DAG. (Default: None)
1326
+ allowed_states : List[str]
1327
+ Iterable of allowed states, (Default: ['success'])
1328
+ failed_states : List[str]
1329
+ Iterable of failed or dis-allowed states. (Default: None)
1330
+ execution_delta : datetime.timedelta
1331
+ time difference with the previous execution to look at,
1332
+ the default is the same logical date as the current task or DAG. (Default: None)
1333
+ check_existence: bool
1334
+ Set to True to check if the external task exists or check if
1335
+ the DAG to wait for exists. (Default: True)
1332
1336
  """
1333
1337
  ...
1334
1338