metaflow-stubs 2.16.4__py2.py3-none-any.whl → 2.16.5__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of metaflow-stubs might be problematic. Click here for more details.

Files changed (166) hide show
  1. metaflow-stubs/__init__.pyi +629 -629
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +6 -6
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +5 -5
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/multicore_utils.pyi +2 -2
  23. metaflow-stubs/packaging_sys/__init__.pyi +5 -5
  24. metaflow-stubs/packaging_sys/backend.pyi +4 -4
  25. metaflow-stubs/packaging_sys/distribution_support.pyi +5 -5
  26. metaflow-stubs/packaging_sys/tar_backend.pyi +6 -6
  27. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  28. metaflow-stubs/packaging_sys/v1.pyi +4 -4
  29. metaflow-stubs/parameters.pyi +4 -4
  30. metaflow-stubs/plugins/__init__.pyi +16 -16
  31. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  33. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  34. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  35. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  37. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  38. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  39. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/argo/argo_client.pyi +3 -3
  41. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  42. metaflow-stubs/plugins/argo/argo_workflows.pyi +5 -5
  43. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  44. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  45. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +22 -4
  46. metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
  47. metaflow-stubs/plugins/aws/__init__.pyi +3 -3
  48. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  50. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  51. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  52. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  53. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  54. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  55. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  56. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  58. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  59. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  60. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  61. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  62. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  63. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  64. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +15 -4
  65. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  67. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  68. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  69. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  70. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  71. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  72. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  74. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  77. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  79. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  81. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  84. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  85. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  86. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  87. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  92. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  93. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  94. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  95. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  96. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  104. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  105. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  106. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  107. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  108. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  110. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  112. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  113. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  114. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  115. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  116. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  119. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  121. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  122. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  123. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  124. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  125. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  126. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  127. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  128. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  129. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  130. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  131. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  132. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  133. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  134. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  135. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  136. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  138. metaflow-stubs/pylint_wrapper.pyi +2 -2
  139. metaflow-stubs/runner/__init__.pyi +2 -2
  140. metaflow-stubs/runner/deployer.pyi +29 -4
  141. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  142. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  143. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  144. metaflow-stubs/runner/nbrun.pyi +2 -2
  145. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  146. metaflow-stubs/runner/utils.pyi +4 -4
  147. metaflow-stubs/system/__init__.pyi +2 -2
  148. metaflow-stubs/system/system_logger.pyi +3 -3
  149. metaflow-stubs/system/system_monitor.pyi +2 -2
  150. metaflow-stubs/tagging_util.pyi +2 -2
  151. metaflow-stubs/tuple_util.pyi +2 -2
  152. metaflow-stubs/user_configs/__init__.pyi +2 -2
  153. metaflow-stubs/user_configs/config_options.pyi +3 -3
  154. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  155. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  156. metaflow-stubs/user_decorators/common.pyi +2 -2
  157. metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
  158. metaflow-stubs/user_decorators/mutable_step.pyi +6 -6
  159. metaflow-stubs/user_decorators/user_flow_decorator.pyi +4 -4
  160. metaflow-stubs/user_decorators/user_step_decorator.pyi +7 -7
  161. metaflow-stubs/version.pyi +2 -2
  162. {metaflow_stubs-2.16.4.dist-info → metaflow_stubs-2.16.5.dist-info}/METADATA +2 -2
  163. metaflow_stubs-2.16.5.dist-info/RECORD +166 -0
  164. metaflow_stubs-2.16.4.dist-info/RECORD +0 -166
  165. {metaflow_stubs-2.16.4.dist-info → metaflow_stubs-2.16.5.dist-info}/WHEEL +0 -0
  166. {metaflow_stubs-2.16.4.dist-info → metaflow_stubs-2.16.5.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.16.4 #
4
- # Generated on 2025-07-17T18:30:30.670730 #
3
+ # MF version: 2.16.5 #
4
+ # Generated on 2025-07-21T20:37:22.333694 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -39,16 +39,16 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
39
39
  from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
+ from . import events as events
42
43
  from . import metaflow_git as metaflow_git
43
44
  from . import tuple_util as tuple_util
44
- from . import events as events
45
45
  from . import runner as runner
46
46
  from . import plugins as plugins
47
47
  from .plugins.datatools.s3.s3 import S3 as S3
48
48
  from . import includefile as includefile
49
49
  from .includefile import IncludeFile as IncludeFile
50
- from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
51
50
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
51
+ from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
52
52
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
53
53
  from . import cards as cards
54
54
  from . import client as client
@@ -152,6 +152,57 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
152
152
  """
153
153
  ...
154
154
 
155
+ @typing.overload
156
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
157
+ """
158
+ Specifies that the step will success under all circumstances.
159
+
160
+ The decorator will create an optional artifact, specified by `var`, which
161
+ contains the exception raised. You can use it to detect the presence
162
+ of errors, indicating that all happy-path artifacts produced by the step
163
+ are missing.
164
+
165
+
166
+ Parameters
167
+ ----------
168
+ var : str, optional, default None
169
+ Name of the artifact in which to store the caught exception.
170
+ If not specified, the exception is not stored.
171
+ print_exception : bool, default True
172
+ Determines whether or not the exception is printed to
173
+ stdout when caught.
174
+ """
175
+ ...
176
+
177
+ @typing.overload
178
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
179
+ ...
180
+
181
+ @typing.overload
182
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
183
+ ...
184
+
185
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
186
+ """
187
+ Specifies that the step will success under all circumstances.
188
+
189
+ The decorator will create an optional artifact, specified by `var`, which
190
+ contains the exception raised. You can use it to detect the presence
191
+ of errors, indicating that all happy-path artifacts produced by the step
192
+ are missing.
193
+
194
+
195
+ Parameters
196
+ ----------
197
+ var : str, optional, default None
198
+ Name of the artifact in which to store the caught exception.
199
+ If not specified, the exception is not stored.
200
+ print_exception : bool, default True
201
+ Determines whether or not the exception is printed to
202
+ stdout when caught.
203
+ """
204
+ ...
205
+
155
206
  @typing.overload
156
207
  def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
157
208
  """
@@ -231,57 +282,6 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
231
282
  """
232
283
  ...
233
284
 
234
- @typing.overload
235
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
236
- """
237
- Specifies that the step will success under all circumstances.
238
-
239
- The decorator will create an optional artifact, specified by `var`, which
240
- contains the exception raised. You can use it to detect the presence
241
- of errors, indicating that all happy-path artifacts produced by the step
242
- are missing.
243
-
244
-
245
- Parameters
246
- ----------
247
- var : str, optional, default None
248
- Name of the artifact in which to store the caught exception.
249
- If not specified, the exception is not stored.
250
- print_exception : bool, default True
251
- Determines whether or not the exception is printed to
252
- stdout when caught.
253
- """
254
- ...
255
-
256
- @typing.overload
257
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
258
- ...
259
-
260
- @typing.overload
261
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
262
- ...
263
-
264
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
265
- """
266
- Specifies that the step will success under all circumstances.
267
-
268
- The decorator will create an optional artifact, specified by `var`, which
269
- contains the exception raised. You can use it to detect the presence
270
- of errors, indicating that all happy-path artifacts produced by the step
271
- are missing.
272
-
273
-
274
- Parameters
275
- ----------
276
- var : str, optional, default None
277
- Name of the artifact in which to store the caught exception.
278
- If not specified, the exception is not stored.
279
- print_exception : bool, default True
280
- Determines whether or not the exception is printed to
281
- stdout when caught.
282
- """
283
- ...
284
-
285
285
  @typing.overload
286
286
  def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
287
287
  """
@@ -302,90 +302,53 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
302
302
  ...
303
303
 
304
304
  @typing.overload
305
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
306
- """
307
- Specifies environment variables to be set prior to the execution of a step.
308
-
309
-
310
- Parameters
311
- ----------
312
- vars : Dict[str, str], default {}
313
- Dictionary of environment variables to set.
314
- """
315
- ...
316
-
317
- @typing.overload
318
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
319
- ...
320
-
321
- @typing.overload
322
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
323
- ...
324
-
325
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
326
- """
327
- Specifies environment variables to be set prior to the execution of a step.
328
-
329
-
330
- Parameters
331
- ----------
332
- vars : Dict[str, str], default {}
333
- Dictionary of environment variables to set.
334
- """
335
- ...
336
-
337
- @typing.overload
338
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
305
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
339
306
  """
340
- Specifies the number of times the task corresponding
341
- to a step needs to be retried.
342
-
343
- This decorator is useful for handling transient errors, such as networking issues.
344
- If your task contains operations that can't be retried safely, e.g. database updates,
345
- it is advisable to annotate it with `@retry(times=0)`.
307
+ Specifies the PyPI packages for the step.
346
308
 
347
- This can be used in conjunction with the `@catch` decorator. The `@catch`
348
- decorator will execute a no-op task after all retries have been exhausted,
349
- ensuring that the flow execution can continue.
309
+ Information in this decorator will augment any
310
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
311
+ you can use `@pypi_base` to set packages required by all
312
+ steps and use `@pypi` to specify step-specific overrides.
350
313
 
351
314
 
352
315
  Parameters
353
316
  ----------
354
- times : int, default 3
355
- Number of times to retry this task.
356
- minutes_between_retries : int, default 2
357
- Number of minutes between retries.
317
+ packages : Dict[str, str], default: {}
318
+ Packages to use for this step. The key is the name of the package
319
+ and the value is the version to use.
320
+ python : str, optional, default: None
321
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
322
+ that the version used will correspond to the version of the Python interpreter used to start the run.
358
323
  """
359
324
  ...
360
325
 
361
326
  @typing.overload
362
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
327
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
363
328
  ...
364
329
 
365
330
  @typing.overload
366
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
331
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
367
332
  ...
368
333
 
369
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
334
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
370
335
  """
371
- Specifies the number of times the task corresponding
372
- to a step needs to be retried.
373
-
374
- This decorator is useful for handling transient errors, such as networking issues.
375
- If your task contains operations that can't be retried safely, e.g. database updates,
376
- it is advisable to annotate it with `@retry(times=0)`.
336
+ Specifies the PyPI packages for the step.
377
337
 
378
- This can be used in conjunction with the `@catch` decorator. The `@catch`
379
- decorator will execute a no-op task after all retries have been exhausted,
380
- ensuring that the flow execution can continue.
338
+ Information in this decorator will augment any
339
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
340
+ you can use `@pypi_base` to set packages required by all
341
+ steps and use `@pypi` to specify step-specific overrides.
381
342
 
382
343
 
383
344
  Parameters
384
345
  ----------
385
- times : int, default 3
386
- Number of times to retry this task.
387
- minutes_between_retries : int, default 2
388
- Number of minutes between retries.
346
+ packages : Dict[str, str], default: {}
347
+ Packages to use for this step. The key is the name of the package
348
+ and the value is the version to use.
349
+ python : str, optional, default: None
350
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
351
+ that the version used will correspond to the version of the Python interpreter used to start the run.
389
352
  """
390
353
  ...
391
354
 
@@ -429,57 +392,9 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
429
392
  ...
430
393
 
431
394
  @typing.overload
432
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
433
- """
434
- Creates a human-readable report, a Metaflow Card, after this step completes.
435
-
436
- Note that you may add multiple `@card` decorators in a step with different parameters.
437
-
438
-
439
- Parameters
440
- ----------
441
- type : str, default 'default'
442
- Card type.
443
- id : str, optional, default None
444
- If multiple cards are present, use this id to identify this card.
445
- options : Dict[str, Any], default {}
446
- Options passed to the card. The contents depend on the card type.
447
- timeout : int, default 45
448
- Interrupt reporting if it takes more than this many seconds.
449
- """
450
- ...
451
-
452
- @typing.overload
453
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
454
- ...
455
-
456
- @typing.overload
457
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
458
- ...
459
-
460
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
461
- """
462
- Creates a human-readable report, a Metaflow Card, after this step completes.
463
-
464
- Note that you may add multiple `@card` decorators in a step with different parameters.
465
-
466
-
467
- Parameters
468
- ----------
469
- type : str, default 'default'
470
- Card type.
471
- id : str, optional, default None
472
- If multiple cards are present, use this id to identify this card.
473
- options : Dict[str, Any], default {}
474
- Options passed to the card. The contents depend on the card type.
475
- timeout : int, default 45
476
- Interrupt reporting if it takes more than this many seconds.
477
- """
478
- ...
479
-
480
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
395
+ def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
481
396
  """
482
- Specifies that this step should execute on Kubernetes.
397
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
483
398
 
484
399
 
485
400
  Parameters
@@ -487,217 +402,9 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
487
402
  cpu : int, default 1
488
403
  Number of CPUs required for this step. If `@resources` is
489
404
  also present, the maximum value from all decorators is used.
490
- memory : int, default 4096
491
- Memory size (in MB) required for this step. If
492
- `@resources` is also present, the maximum value from all decorators is
493
- used.
494
- disk : int, default 10240
495
- Disk size (in MB) required for this step. If
496
- `@resources` is also present, the maximum value from all decorators is
497
- used.
498
- image : str, optional, default None
499
- Docker image to use when launching on Kubernetes. If not specified, and
500
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
501
- not, a default Docker image mapping to the current version of Python is used.
502
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
503
- If given, the imagePullPolicy to be applied to the Docker image of the step.
504
- image_pull_secrets: List[str], default []
505
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
506
- Kubernetes image pull secrets to use when pulling container images
507
- in Kubernetes.
508
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
509
- Kubernetes service account to use when launching pod in Kubernetes.
510
- secrets : List[str], optional, default None
511
- Kubernetes secrets to use when launching pod in Kubernetes. These
512
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
513
- in Metaflow configuration.
514
- node_selector: Union[Dict[str,str], str], optional, default None
515
- Kubernetes node selector(s) to apply to the pod running the task.
516
- Can be passed in as a comma separated string of values e.g.
517
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
518
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
519
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
520
- Kubernetes namespace to use when launching pod in Kubernetes.
521
- gpu : int, optional, default None
522
- Number of GPUs required for this step. A value of zero implies that
523
- the scheduled node should not have GPUs.
524
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
525
- The vendor of the GPUs to be used for this step.
526
- tolerations : List[str], default []
527
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
528
- Kubernetes tolerations to use when launching pod in Kubernetes.
529
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
530
- Kubernetes labels to use when launching pod in Kubernetes.
531
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
532
- Kubernetes annotations to use when launching pod in Kubernetes.
533
- use_tmpfs : bool, default False
534
- This enables an explicit tmpfs mount for this step.
535
- tmpfs_tempdir : bool, default True
536
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
537
- tmpfs_size : int, optional, default: None
538
- The value for the size (in MiB) of the tmpfs mount for this step.
539
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
540
- memory allocated for this step.
541
- tmpfs_path : str, optional, default /metaflow_temp
542
- Path to tmpfs mount for this step.
543
- persistent_volume_claims : Dict[str, str], optional, default None
544
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
545
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
546
- shared_memory: int, optional
547
- Shared memory size (in MiB) required for this step
548
- port: int, optional
549
- Port number to specify in the Kubernetes job object
550
- compute_pool : str, optional, default None
551
- Compute pool to be used for for this step.
552
- If not specified, any accessible compute pool within the perimeter is used.
553
- hostname_resolution_timeout: int, default 10 * 60
554
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
555
- Only applicable when @parallel is used.
556
- qos: str, default: Burstable
557
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
558
-
559
- security_context: Dict[str, Any], optional, default None
560
- Container security context. Applies to the task container. Allows the following keys:
561
- - privileged: bool, optional, default None
562
- - allow_privilege_escalation: bool, optional, default None
563
- - run_as_user: int, optional, default None
564
- - run_as_group: int, optional, default None
565
- - run_as_non_root: bool, optional, default None
566
- """
567
- ...
568
-
569
- @typing.overload
570
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
571
- """
572
- Specifies the Conda environment for the step.
573
-
574
- Information in this decorator will augment any
575
- attributes set in the `@conda_base` flow-level decorator. Hence,
576
- you can use `@conda_base` to set packages required by all
577
- steps and use `@conda` to specify step-specific overrides.
578
-
579
-
580
- Parameters
581
- ----------
582
- packages : Dict[str, str], default {}
583
- Packages to use for this step. The key is the name of the package
584
- and the value is the version to use.
585
- libraries : Dict[str, str], default {}
586
- Supported for backward compatibility. When used with packages, packages will take precedence.
587
- python : str, optional, default None
588
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
589
- that the version used will correspond to the version of the Python interpreter used to start the run.
590
- disabled : bool, default False
591
- If set to True, disables @conda.
592
- """
593
- ...
594
-
595
- @typing.overload
596
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
597
- ...
598
-
599
- @typing.overload
600
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
601
- ...
602
-
603
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
604
- """
605
- Specifies the Conda environment for the step.
606
-
607
- Information in this decorator will augment any
608
- attributes set in the `@conda_base` flow-level decorator. Hence,
609
- you can use `@conda_base` to set packages required by all
610
- steps and use `@conda` to specify step-specific overrides.
611
-
612
-
613
- Parameters
614
- ----------
615
- packages : Dict[str, str], default {}
616
- Packages to use for this step. The key is the name of the package
617
- and the value is the version to use.
618
- libraries : Dict[str, str], default {}
619
- Supported for backward compatibility. When used with packages, packages will take precedence.
620
- python : str, optional, default None
621
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
622
- that the version used will correspond to the version of the Python interpreter used to start the run.
623
- disabled : bool, default False
624
- If set to True, disables @conda.
625
- """
626
- ...
627
-
628
- @typing.overload
629
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
630
- """
631
- Specifies a timeout for your step.
632
-
633
- This decorator is useful if this step may hang indefinitely.
634
-
635
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
636
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
637
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
638
-
639
- Note that all the values specified in parameters are added together so if you specify
640
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
641
-
642
-
643
- Parameters
644
- ----------
645
- seconds : int, default 0
646
- Number of seconds to wait prior to timing out.
647
- minutes : int, default 0
648
- Number of minutes to wait prior to timing out.
649
- hours : int, default 0
650
- Number of hours to wait prior to timing out.
651
- """
652
- ...
653
-
654
- @typing.overload
655
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
656
- ...
657
-
658
- @typing.overload
659
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
660
- ...
661
-
662
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
663
- """
664
- Specifies a timeout for your step.
665
-
666
- This decorator is useful if this step may hang indefinitely.
667
-
668
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
669
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
670
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
671
-
672
- Note that all the values specified in parameters are added together so if you specify
673
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
674
-
675
-
676
- Parameters
677
- ----------
678
- seconds : int, default 0
679
- Number of seconds to wait prior to timing out.
680
- minutes : int, default 0
681
- Number of minutes to wait prior to timing out.
682
- hours : int, default 0
683
- Number of hours to wait prior to timing out.
684
- """
685
- ...
686
-
687
- @typing.overload
688
- def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
689
- """
690
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
691
-
692
-
693
- Parameters
694
- ----------
695
- cpu : int, default 1
696
- Number of CPUs required for this step. If `@resources` is
697
- also present, the maximum value from all decorators is used.
698
- gpu : int, default 0
699
- Number of GPUs required for this step. If `@resources` is
700
- also present, the maximum value from all decorators is used.
405
+ gpu : int, default 0
406
+ Number of GPUs required for this step. If `@resources` is
407
+ also present, the maximum value from all decorators is used.
701
408
  memory : int, default 4096
702
409
  Memory size (in MB) required for this step. If
703
410
  `@resources` is also present, the maximum value from all decorators is
@@ -834,60 +541,397 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
834
541
  ...
835
542
 
836
543
  @typing.overload
837
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
544
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
838
545
  """
839
- Specifies the PyPI packages for the step.
546
+ Creates a human-readable report, a Metaflow Card, after this step completes.
840
547
 
841
- Information in this decorator will augment any
842
- attributes set in the `@pyi_base` flow-level decorator. Hence,
843
- you can use `@pypi_base` to set packages required by all
844
- steps and use `@pypi` to specify step-specific overrides.
548
+ Note that you may add multiple `@card` decorators in a step with different parameters.
845
549
 
846
550
 
847
551
  Parameters
848
552
  ----------
849
- packages : Dict[str, str], default: {}
850
- Packages to use for this step. The key is the name of the package
851
- and the value is the version to use.
852
- python : str, optional, default: None
853
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
854
- that the version used will correspond to the version of the Python interpreter used to start the run.
553
+ type : str, default 'default'
554
+ Card type.
555
+ id : str, optional, default None
556
+ If multiple cards are present, use this id to identify this card.
557
+ options : Dict[str, Any], default {}
558
+ Options passed to the card. The contents depend on the card type.
559
+ timeout : int, default 45
560
+ Interrupt reporting if it takes more than this many seconds.
855
561
  """
856
562
  ...
857
563
 
858
564
  @typing.overload
859
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
565
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
860
566
  ...
861
567
 
862
568
  @typing.overload
863
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
569
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
864
570
  ...
865
571
 
866
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
572
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
867
573
  """
868
- Specifies the PyPI packages for the step.
574
+ Creates a human-readable report, a Metaflow Card, after this step completes.
869
575
 
870
- Information in this decorator will augment any
871
- attributes set in the `@pyi_base` flow-level decorator. Hence,
872
- you can use `@pypi_base` to set packages required by all
873
- steps and use `@pypi` to specify step-specific overrides.
576
+ Note that you may add multiple `@card` decorators in a step with different parameters.
874
577
 
875
578
 
876
579
  Parameters
877
580
  ----------
878
- packages : Dict[str, str], default: {}
879
- Packages to use for this step. The key is the name of the package
880
- and the value is the version to use.
881
- python : str, optional, default: None
882
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
883
- that the version used will correspond to the version of the Python interpreter used to start the run.
581
+ type : str, default 'default'
582
+ Card type.
583
+ id : str, optional, default None
584
+ If multiple cards are present, use this id to identify this card.
585
+ options : Dict[str, Any], default {}
586
+ Options passed to the card. The contents depend on the card type.
587
+ timeout : int, default 45
588
+ Interrupt reporting if it takes more than this many seconds.
884
589
  """
885
590
  ...
886
591
 
887
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
592
+ @typing.overload
593
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
888
594
  """
889
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
890
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
595
+ Specifies the Conda environment for the step.
596
+
597
+ Information in this decorator will augment any
598
+ attributes set in the `@conda_base` flow-level decorator. Hence,
599
+ you can use `@conda_base` to set packages required by all
600
+ steps and use `@conda` to specify step-specific overrides.
601
+
602
+
603
+ Parameters
604
+ ----------
605
+ packages : Dict[str, str], default {}
606
+ Packages to use for this step. The key is the name of the package
607
+ and the value is the version to use.
608
+ libraries : Dict[str, str], default {}
609
+ Supported for backward compatibility. When used with packages, packages will take precedence.
610
+ python : str, optional, default None
611
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
612
+ that the version used will correspond to the version of the Python interpreter used to start the run.
613
+ disabled : bool, default False
614
+ If set to True, disables @conda.
615
+ """
616
+ ...
617
+
618
+ @typing.overload
619
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
620
+ ...
621
+
622
+ @typing.overload
623
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
624
+ ...
625
+
626
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
627
+ """
628
+ Specifies the Conda environment for the step.
629
+
630
+ Information in this decorator will augment any
631
+ attributes set in the `@conda_base` flow-level decorator. Hence,
632
+ you can use `@conda_base` to set packages required by all
633
+ steps and use `@conda` to specify step-specific overrides.
634
+
635
+
636
+ Parameters
637
+ ----------
638
+ packages : Dict[str, str], default {}
639
+ Packages to use for this step. The key is the name of the package
640
+ and the value is the version to use.
641
+ libraries : Dict[str, str], default {}
642
+ Supported for backward compatibility. When used with packages, packages will take precedence.
643
+ python : str, optional, default None
644
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
645
+ that the version used will correspond to the version of the Python interpreter used to start the run.
646
+ disabled : bool, default False
647
+ If set to True, disables @conda.
648
+ """
649
+ ...
650
+
651
+ @typing.overload
652
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
653
+ """
654
+ Specifies a timeout for your step.
655
+
656
+ This decorator is useful if this step may hang indefinitely.
657
+
658
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
659
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
660
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
661
+
662
+ Note that all the values specified in parameters are added together so if you specify
663
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
664
+
665
+
666
+ Parameters
667
+ ----------
668
+ seconds : int, default 0
669
+ Number of seconds to wait prior to timing out.
670
+ minutes : int, default 0
671
+ Number of minutes to wait prior to timing out.
672
+ hours : int, default 0
673
+ Number of hours to wait prior to timing out.
674
+ """
675
+ ...
676
+
677
+ @typing.overload
678
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
679
+ ...
680
+
681
+ @typing.overload
682
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
683
+ ...
684
+
685
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
686
+ """
687
+ Specifies a timeout for your step.
688
+
689
+ This decorator is useful if this step may hang indefinitely.
690
+
691
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
692
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
693
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
694
+
695
+ Note that all the values specified in parameters are added together so if you specify
696
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
697
+
698
+
699
+ Parameters
700
+ ----------
701
+ seconds : int, default 0
702
+ Number of seconds to wait prior to timing out.
703
+ minutes : int, default 0
704
+ Number of minutes to wait prior to timing out.
705
+ hours : int, default 0
706
+ Number of hours to wait prior to timing out.
707
+ """
708
+ ...
709
+
710
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
711
+ """
712
+ Specifies that this step should execute on Kubernetes.
713
+
714
+
715
+ Parameters
716
+ ----------
717
+ cpu : int, default 1
718
+ Number of CPUs required for this step. If `@resources` is
719
+ also present, the maximum value from all decorators is used.
720
+ memory : int, default 4096
721
+ Memory size (in MB) required for this step. If
722
+ `@resources` is also present, the maximum value from all decorators is
723
+ used.
724
+ disk : int, default 10240
725
+ Disk size (in MB) required for this step. If
726
+ `@resources` is also present, the maximum value from all decorators is
727
+ used.
728
+ image : str, optional, default None
729
+ Docker image to use when launching on Kubernetes. If not specified, and
730
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
731
+ not, a default Docker image mapping to the current version of Python is used.
732
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
733
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
734
+ image_pull_secrets: List[str], default []
735
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
736
+ Kubernetes image pull secrets to use when pulling container images
737
+ in Kubernetes.
738
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
739
+ Kubernetes service account to use when launching pod in Kubernetes.
740
+ secrets : List[str], optional, default None
741
+ Kubernetes secrets to use when launching pod in Kubernetes. These
742
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
743
+ in Metaflow configuration.
744
+ node_selector: Union[Dict[str,str], str], optional, default None
745
+ Kubernetes node selector(s) to apply to the pod running the task.
746
+ Can be passed in as a comma separated string of values e.g.
747
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
748
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
749
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
750
+ Kubernetes namespace to use when launching pod in Kubernetes.
751
+ gpu : int, optional, default None
752
+ Number of GPUs required for this step. A value of zero implies that
753
+ the scheduled node should not have GPUs.
754
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
755
+ The vendor of the GPUs to be used for this step.
756
+ tolerations : List[Dict[str,str]], default []
757
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
758
+ Kubernetes tolerations to use when launching pod in Kubernetes.
759
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
760
+ Kubernetes labels to use when launching pod in Kubernetes.
761
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
762
+ Kubernetes annotations to use when launching pod in Kubernetes.
763
+ use_tmpfs : bool, default False
764
+ This enables an explicit tmpfs mount for this step.
765
+ tmpfs_tempdir : bool, default True
766
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
767
+ tmpfs_size : int, optional, default: None
768
+ The value for the size (in MiB) of the tmpfs mount for this step.
769
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
770
+ memory allocated for this step.
771
+ tmpfs_path : str, optional, default /metaflow_temp
772
+ Path to tmpfs mount for this step.
773
+ persistent_volume_claims : Dict[str, str], optional, default None
774
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
775
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
776
+ shared_memory: int, optional
777
+ Shared memory size (in MiB) required for this step
778
+ port: int, optional
779
+ Port number to specify in the Kubernetes job object
780
+ compute_pool : str, optional, default None
781
+ Compute pool to be used for for this step.
782
+ If not specified, any accessible compute pool within the perimeter is used.
783
+ hostname_resolution_timeout: int, default 10 * 60
784
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
785
+ Only applicable when @parallel is used.
786
+ qos: str, default: Burstable
787
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
788
+
789
+ security_context: Dict[str, Any], optional, default None
790
+ Container security context. Applies to the task container. Allows the following keys:
791
+ - privileged: bool, optional, default None
792
+ - allow_privilege_escalation: bool, optional, default None
793
+ - run_as_user: int, optional, default None
794
+ - run_as_group: int, optional, default None
795
+ - run_as_non_root: bool, optional, default None
796
+ """
797
+ ...
798
+
799
+ @typing.overload
800
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
801
+ """
802
+ Specifies environment variables to be set prior to the execution of a step.
803
+
804
+
805
+ Parameters
806
+ ----------
807
+ vars : Dict[str, str], default {}
808
+ Dictionary of environment variables to set.
809
+ """
810
+ ...
811
+
812
+ @typing.overload
813
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
814
+ ...
815
+
816
+ @typing.overload
817
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
818
+ ...
819
+
820
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
821
+ """
822
+ Specifies environment variables to be set prior to the execution of a step.
823
+
824
+
825
+ Parameters
826
+ ----------
827
+ vars : Dict[str, str], default {}
828
+ Dictionary of environment variables to set.
829
+ """
830
+ ...
831
+
832
+ @typing.overload
833
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
834
+ """
835
+ Specifies the number of times the task corresponding
836
+ to a step needs to be retried.
837
+
838
+ This decorator is useful for handling transient errors, such as networking issues.
839
+ If your task contains operations that can't be retried safely, e.g. database updates,
840
+ it is advisable to annotate it with `@retry(times=0)`.
841
+
842
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
843
+ decorator will execute a no-op task after all retries have been exhausted,
844
+ ensuring that the flow execution can continue.
845
+
846
+
847
+ Parameters
848
+ ----------
849
+ times : int, default 3
850
+ Number of times to retry this task.
851
+ minutes_between_retries : int, default 2
852
+ Number of minutes between retries.
853
+ """
854
+ ...
855
+
856
+ @typing.overload
857
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
858
+ ...
859
+
860
+ @typing.overload
861
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
862
+ ...
863
+
864
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
865
+ """
866
+ Specifies the number of times the task corresponding
867
+ to a step needs to be retried.
868
+
869
+ This decorator is useful for handling transient errors, such as networking issues.
870
+ If your task contains operations that can't be retried safely, e.g. database updates,
871
+ it is advisable to annotate it with `@retry(times=0)`.
872
+
873
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
874
+ decorator will execute a no-op task after all retries have been exhausted,
875
+ ensuring that the flow execution can continue.
876
+
877
+
878
+ Parameters
879
+ ----------
880
+ times : int, default 3
881
+ Number of times to retry this task.
882
+ minutes_between_retries : int, default 2
883
+ Number of minutes between retries.
884
+ """
885
+ ...
886
+
887
+ @typing.overload
888
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
889
+ """
890
+ Specifies the PyPI packages for all steps of the flow.
891
+
892
+ Use `@pypi_base` to set common packages required by all
893
+ steps and use `@pypi` to specify step-specific overrides.
894
+
895
+ Parameters
896
+ ----------
897
+ packages : Dict[str, str], default: {}
898
+ Packages to use for this flow. The key is the name of the package
899
+ and the value is the version to use.
900
+ python : str, optional, default: None
901
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
902
+ that the version used will correspond to the version of the Python interpreter used to start the run.
903
+ """
904
+ ...
905
+
906
+ @typing.overload
907
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
908
+ ...
909
+
910
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
911
+ """
912
+ Specifies the PyPI packages for all steps of the flow.
913
+
914
+ Use `@pypi_base` to set common packages required by all
915
+ steps and use `@pypi` to specify step-specific overrides.
916
+
917
+ Parameters
918
+ ----------
919
+ packages : Dict[str, str], default: {}
920
+ Packages to use for this flow. The key is the name of the package
921
+ and the value is the version to use.
922
+ python : str, optional, default: None
923
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
924
+ that the version used will correspond to the version of the Python interpreter used to start the run.
925
+ """
926
+ ...
927
+
928
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
929
+ """
930
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
931
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
932
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
933
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
934
+ starts only after all sensors finish.
891
935
 
892
936
 
893
937
  Parameters
@@ -909,21 +953,18 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
909
953
  Name of the sensor on Airflow
910
954
  description : str
911
955
  Description of sensor in the Airflow UI
912
- external_dag_id : str
913
- The dag_id that contains the task you want to wait for.
914
- external_task_ids : List[str]
915
- The list of task_ids that you want to wait for.
916
- If None (default value) the sensor waits for the DAG. (Default: None)
917
- allowed_states : List[str]
918
- Iterable of allowed states, (Default: ['success'])
919
- failed_states : List[str]
920
- Iterable of failed or dis-allowed states. (Default: None)
921
- execution_delta : datetime.timedelta
922
- time difference with the previous execution to look at,
923
- the default is the same logical date as the current task or DAG. (Default: None)
924
- check_existence: bool
925
- Set to True to check if the external task exists or check if
926
- the DAG to wait for exists. (Default: True)
956
+ bucket_key : Union[str, List[str]]
957
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
958
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
959
+ bucket_name : str
960
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
961
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
962
+ wildcard_match : bool
963
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
964
+ aws_conn_id : str
965
+ a reference to the s3 connection on Airflow. (Default: None)
966
+ verify : bool
967
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
927
968
  """
928
969
  ...
929
970
 
@@ -978,107 +1019,177 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
978
1019
  """
979
1020
  ...
980
1021
 
1022
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1023
+ """
1024
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1025
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1026
+
1027
+
1028
+ Parameters
1029
+ ----------
1030
+ timeout : int
1031
+ Time, in seconds before the task times out and fails. (Default: 3600)
1032
+ poke_interval : int
1033
+ Time in seconds that the job should wait in between each try. (Default: 60)
1034
+ mode : str
1035
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1036
+ exponential_backoff : bool
1037
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1038
+ pool : str
1039
+ the slot pool this task should run in,
1040
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1041
+ soft_fail : bool
1042
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1043
+ name : str
1044
+ Name of the sensor on Airflow
1045
+ description : str
1046
+ Description of sensor in the Airflow UI
1047
+ external_dag_id : str
1048
+ The dag_id that contains the task you want to wait for.
1049
+ external_task_ids : List[str]
1050
+ The list of task_ids that you want to wait for.
1051
+ If None (default value) the sensor waits for the DAG. (Default: None)
1052
+ allowed_states : List[str]
1053
+ Iterable of allowed states, (Default: ['success'])
1054
+ failed_states : List[str]
1055
+ Iterable of failed or dis-allowed states. (Default: None)
1056
+ execution_delta : datetime.timedelta
1057
+ time difference with the previous execution to look at,
1058
+ the default is the same logical date as the current task or DAG. (Default: None)
1059
+ check_existence: bool
1060
+ Set to True to check if the external task exists or check if
1061
+ the DAG to wait for exists. (Default: True)
1062
+ """
1063
+ ...
1064
+
981
1065
  @typing.overload
982
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1066
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
983
1067
  """
984
- Specifies the flow(s) that this flow depends on.
1068
+ Specifies the event(s) that this flow depends on.
985
1069
 
986
1070
  ```
987
- @trigger_on_finish(flow='FooFlow')
1071
+ @trigger(event='foo')
988
1072
  ```
989
1073
  or
990
1074
  ```
991
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1075
+ @trigger(events=['foo', 'bar'])
992
1076
  ```
993
- This decorator respects the @project decorator and triggers the flow
994
- when upstream runs within the same namespace complete successfully
995
1077
 
996
- Additionally, you can specify project aware upstream flow dependencies
997
- by specifying the fully qualified project_flow_name.
1078
+ Additionally, you can specify the parameter mappings
1079
+ to map event payload to Metaflow parameters for the flow.
998
1080
  ```
999
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1081
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1000
1082
  ```
1001
1083
  or
1002
1084
  ```
1003
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1085
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1086
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1004
1087
  ```
1005
1088
 
1006
- You can also specify just the project or project branch (other values will be
1007
- inferred from the current project or project branch):
1089
+ 'parameters' can also be a list of strings and tuples like so:
1008
1090
  ```
1009
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1091
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1092
+ ```
1093
+ This is equivalent to:
1094
+ ```
1095
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1010
1096
  ```
1011
-
1012
- Note that `branch` is typically one of:
1013
- - `prod`
1014
- - `user.bob`
1015
- - `test.my_experiment`
1016
- - `prod.staging`
1017
1097
 
1018
1098
 
1019
1099
  Parameters
1020
1100
  ----------
1021
- flow : Union[str, Dict[str, str]], optional, default None
1022
- Upstream flow dependency for this flow.
1023
- flows : List[Union[str, Dict[str, str]]], default []
1024
- Upstream flow dependencies for this flow.
1101
+ event : Union[str, Dict[str, Any]], optional, default None
1102
+ Event dependency for this flow.
1103
+ events : List[Union[str, Dict[str, Any]]], default []
1104
+ Events dependency for this flow.
1025
1105
  options : Dict[str, Any], default {}
1026
1106
  Backend-specific configuration for tuning eventing behavior.
1027
1107
  """
1028
1108
  ...
1029
1109
 
1030
1110
  @typing.overload
1031
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1111
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1032
1112
  ...
1033
1113
 
1034
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1114
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1035
1115
  """
1036
- Specifies the flow(s) that this flow depends on.
1116
+ Specifies the event(s) that this flow depends on.
1037
1117
 
1038
1118
  ```
1039
- @trigger_on_finish(flow='FooFlow')
1119
+ @trigger(event='foo')
1040
1120
  ```
1041
1121
  or
1042
1122
  ```
1043
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1123
+ @trigger(events=['foo', 'bar'])
1044
1124
  ```
1045
- This decorator respects the @project decorator and triggers the flow
1046
- when upstream runs within the same namespace complete successfully
1047
1125
 
1048
- Additionally, you can specify project aware upstream flow dependencies
1049
- by specifying the fully qualified project_flow_name.
1126
+ Additionally, you can specify the parameter mappings
1127
+ to map event payload to Metaflow parameters for the flow.
1050
1128
  ```
1051
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1129
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1052
1130
  ```
1053
1131
  or
1054
1132
  ```
1055
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1133
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1134
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1056
1135
  ```
1057
1136
 
1058
- You can also specify just the project or project branch (other values will be
1059
- inferred from the current project or project branch):
1137
+ 'parameters' can also be a list of strings and tuples like so:
1138
+ ```
1139
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1140
+ ```
1141
+ This is equivalent to:
1060
1142
  ```
1061
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1143
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1062
1144
  ```
1063
1145
 
1064
- Note that `branch` is typically one of:
1065
- - `prod`
1066
- - `user.bob`
1067
- - `test.my_experiment`
1068
- - `prod.staging`
1069
-
1070
1146
 
1071
1147
  Parameters
1072
1148
  ----------
1073
- flow : Union[str, Dict[str, str]], optional, default None
1074
- Upstream flow dependency for this flow.
1075
- flows : List[Union[str, Dict[str, str]]], default []
1076
- Upstream flow dependencies for this flow.
1149
+ event : Union[str, Dict[str, Any]], optional, default None
1150
+ Event dependency for this flow.
1151
+ events : List[Union[str, Dict[str, Any]]], default []
1152
+ Events dependency for this flow.
1077
1153
  options : Dict[str, Any], default {}
1078
1154
  Backend-specific configuration for tuning eventing behavior.
1079
1155
  """
1080
1156
  ...
1081
1157
 
1158
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1159
+ """
1160
+ Specifies what flows belong to the same project.
1161
+
1162
+ A project-specific namespace is created for all flows that
1163
+ use the same `@project(name)`.
1164
+
1165
+
1166
+ Parameters
1167
+ ----------
1168
+ name : str
1169
+ Project name. Make sure that the name is unique amongst all
1170
+ projects that use the same production scheduler. The name may
1171
+ contain only lowercase alphanumeric characters and underscores.
1172
+
1173
+ branch : Optional[str], default None
1174
+ The branch to use. If not specified, the branch is set to
1175
+ `user.<username>` unless `production` is set to `True`. This can
1176
+ also be set on the command line using `--branch` as a top-level option.
1177
+ It is an error to specify `branch` in the decorator and on the command line.
1178
+
1179
+ production : bool, default False
1180
+ Whether or not the branch is the production branch. This can also be set on the
1181
+ command line using `--production` as a top-level option. It is an error to specify
1182
+ `production` in the decorator and on the command line.
1183
+ The project branch name will be:
1184
+ - if `branch` is specified:
1185
+ - if `production` is True: `prod.<branch>`
1186
+ - if `production` is False: `test.<branch>`
1187
+ - if `branch` is not specified:
1188
+ - if `production` is True: `prod`
1189
+ - if `production` is False: `user.<username>`
1190
+ """
1191
+ ...
1192
+
1082
1193
  @typing.overload
1083
1194
  def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1084
1195
  """
@@ -1130,213 +1241,102 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1130
1241
  """
1131
1242
  ...
1132
1243
 
1133
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1134
- """
1135
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1136
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1137
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1138
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1139
- starts only after all sensors finish.
1140
-
1141
-
1142
- Parameters
1143
- ----------
1144
- timeout : int
1145
- Time, in seconds before the task times out and fails. (Default: 3600)
1146
- poke_interval : int
1147
- Time in seconds that the job should wait in between each try. (Default: 60)
1148
- mode : str
1149
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1150
- exponential_backoff : bool
1151
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1152
- pool : str
1153
- the slot pool this task should run in,
1154
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1155
- soft_fail : bool
1156
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1157
- name : str
1158
- Name of the sensor on Airflow
1159
- description : str
1160
- Description of sensor in the Airflow UI
1161
- bucket_key : Union[str, List[str]]
1162
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1163
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1164
- bucket_name : str
1165
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1166
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1167
- wildcard_match : bool
1168
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1169
- aws_conn_id : str
1170
- a reference to the s3 connection on Airflow. (Default: None)
1171
- verify : bool
1172
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1173
- """
1174
- ...
1175
-
1176
- @typing.overload
1177
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1178
- """
1179
- Specifies the PyPI packages for all steps of the flow.
1180
-
1181
- Use `@pypi_base` to set common packages required by all
1182
- steps and use `@pypi` to specify step-specific overrides.
1183
-
1184
- Parameters
1185
- ----------
1186
- packages : Dict[str, str], default: {}
1187
- Packages to use for this flow. The key is the name of the package
1188
- and the value is the version to use.
1189
- python : str, optional, default: None
1190
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1191
- that the version used will correspond to the version of the Python interpreter used to start the run.
1192
- """
1193
- ...
1194
-
1195
- @typing.overload
1196
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1197
- ...
1198
-
1199
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1200
- """
1201
- Specifies the PyPI packages for all steps of the flow.
1202
-
1203
- Use `@pypi_base` to set common packages required by all
1204
- steps and use `@pypi` to specify step-specific overrides.
1205
-
1206
- Parameters
1207
- ----------
1208
- packages : Dict[str, str], default: {}
1209
- Packages to use for this flow. The key is the name of the package
1210
- and the value is the version to use.
1211
- python : str, optional, default: None
1212
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1213
- that the version used will correspond to the version of the Python interpreter used to start the run.
1214
- """
1215
- ...
1216
-
1217
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1218
- """
1219
- Specifies what flows belong to the same project.
1220
-
1221
- A project-specific namespace is created for all flows that
1222
- use the same `@project(name)`.
1223
-
1224
-
1225
- Parameters
1226
- ----------
1227
- name : str
1228
- Project name. Make sure that the name is unique amongst all
1229
- projects that use the same production scheduler. The name may
1230
- contain only lowercase alphanumeric characters and underscores.
1231
-
1232
- branch : Optional[str], default None
1233
- The branch to use. If not specified, the branch is set to
1234
- `user.<username>` unless `production` is set to `True`. This can
1235
- also be set on the command line using `--branch` as a top-level option.
1236
- It is an error to specify `branch` in the decorator and on the command line.
1237
-
1238
- production : bool, default False
1239
- Whether or not the branch is the production branch. This can also be set on the
1240
- command line using `--production` as a top-level option. It is an error to specify
1241
- `production` in the decorator and on the command line.
1242
- The project branch name will be:
1243
- - if `branch` is specified:
1244
- - if `production` is True: `prod.<branch>`
1245
- - if `production` is False: `test.<branch>`
1246
- - if `branch` is not specified:
1247
- - if `production` is True: `prod`
1248
- - if `production` is False: `user.<username>`
1249
- """
1250
- ...
1251
-
1252
1244
  @typing.overload
1253
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1245
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1254
1246
  """
1255
- Specifies the event(s) that this flow depends on.
1247
+ Specifies the flow(s) that this flow depends on.
1256
1248
 
1257
1249
  ```
1258
- @trigger(event='foo')
1250
+ @trigger_on_finish(flow='FooFlow')
1259
1251
  ```
1260
1252
  or
1261
1253
  ```
1262
- @trigger(events=['foo', 'bar'])
1254
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1263
1255
  ```
1256
+ This decorator respects the @project decorator and triggers the flow
1257
+ when upstream runs within the same namespace complete successfully
1264
1258
 
1265
- Additionally, you can specify the parameter mappings
1266
- to map event payload to Metaflow parameters for the flow.
1259
+ Additionally, you can specify project aware upstream flow dependencies
1260
+ by specifying the fully qualified project_flow_name.
1267
1261
  ```
1268
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1262
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1269
1263
  ```
1270
1264
  or
1271
1265
  ```
1272
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1273
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1266
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1274
1267
  ```
1275
1268
 
1276
- 'parameters' can also be a list of strings and tuples like so:
1277
- ```
1278
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1279
- ```
1280
- This is equivalent to:
1269
+ You can also specify just the project or project branch (other values will be
1270
+ inferred from the current project or project branch):
1281
1271
  ```
1282
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1272
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1283
1273
  ```
1284
1274
 
1275
+ Note that `branch` is typically one of:
1276
+ - `prod`
1277
+ - `user.bob`
1278
+ - `test.my_experiment`
1279
+ - `prod.staging`
1280
+
1285
1281
 
1286
1282
  Parameters
1287
1283
  ----------
1288
- event : Union[str, Dict[str, Any]], optional, default None
1289
- Event dependency for this flow.
1290
- events : List[Union[str, Dict[str, Any]]], default []
1291
- Events dependency for this flow.
1284
+ flow : Union[str, Dict[str, str]], optional, default None
1285
+ Upstream flow dependency for this flow.
1286
+ flows : List[Union[str, Dict[str, str]]], default []
1287
+ Upstream flow dependencies for this flow.
1292
1288
  options : Dict[str, Any], default {}
1293
1289
  Backend-specific configuration for tuning eventing behavior.
1294
1290
  """
1295
1291
  ...
1296
1292
 
1297
1293
  @typing.overload
1298
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1294
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1299
1295
  ...
1300
1296
 
1301
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1297
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1302
1298
  """
1303
- Specifies the event(s) that this flow depends on.
1299
+ Specifies the flow(s) that this flow depends on.
1304
1300
 
1305
1301
  ```
1306
- @trigger(event='foo')
1302
+ @trigger_on_finish(flow='FooFlow')
1307
1303
  ```
1308
1304
  or
1309
1305
  ```
1310
- @trigger(events=['foo', 'bar'])
1306
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1311
1307
  ```
1308
+ This decorator respects the @project decorator and triggers the flow
1309
+ when upstream runs within the same namespace complete successfully
1312
1310
 
1313
- Additionally, you can specify the parameter mappings
1314
- to map event payload to Metaflow parameters for the flow.
1311
+ Additionally, you can specify project aware upstream flow dependencies
1312
+ by specifying the fully qualified project_flow_name.
1315
1313
  ```
1316
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1314
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1317
1315
  ```
1318
1316
  or
1319
1317
  ```
1320
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1321
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1318
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1322
1319
  ```
1323
1320
 
1324
- 'parameters' can also be a list of strings and tuples like so:
1325
- ```
1326
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1327
- ```
1328
- This is equivalent to:
1321
+ You can also specify just the project or project branch (other values will be
1322
+ inferred from the current project or project branch):
1329
1323
  ```
1330
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1324
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1331
1325
  ```
1332
1326
 
1327
+ Note that `branch` is typically one of:
1328
+ - `prod`
1329
+ - `user.bob`
1330
+ - `test.my_experiment`
1331
+ - `prod.staging`
1332
+
1333
1333
 
1334
1334
  Parameters
1335
1335
  ----------
1336
- event : Union[str, Dict[str, Any]], optional, default None
1337
- Event dependency for this flow.
1338
- events : List[Union[str, Dict[str, Any]]], default []
1339
- Events dependency for this flow.
1336
+ flow : Union[str, Dict[str, str]], optional, default None
1337
+ Upstream flow dependency for this flow.
1338
+ flows : List[Union[str, Dict[str, str]]], default []
1339
+ Upstream flow dependencies for this flow.
1340
1340
  options : Dict[str, Any], default {}
1341
1341
  Backend-specific configuration for tuning eventing behavior.
1342
1342
  """