metaflow-stubs 2.15.1__py2.py3-none-any.whl → 2.15.2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. metaflow-stubs/__init__.pyi +451 -451
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +17 -17
  21. metaflow-stubs/multicore_utils.pyi +2 -2
  22. metaflow-stubs/parameters.pyi +3 -3
  23. metaflow-stubs/plugins/__init__.pyi +11 -11
  24. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  31. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  32. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  35. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  37. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  38. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  39. metaflow-stubs/plugins/aws/__init__.pyi +3 -3
  40. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  47. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  48. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  57. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  61. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  62. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  63. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  64. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  77. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  78. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  79. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  80. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  82. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  83. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  84. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  85. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  86. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  87. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  88. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  89. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  90. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  94. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  95. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  96. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  97. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  98. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  100. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  101. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  102. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  106. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  107. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  111. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  112. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  114. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  115. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  118. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  119. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  121. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  122. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  123. metaflow-stubs/pylint_wrapper.pyi +2 -2
  124. metaflow-stubs/runner/__init__.pyi +2 -2
  125. metaflow-stubs/runner/deployer.pyi +5 -5
  126. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  127. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  128. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  129. metaflow-stubs/runner/nbrun.pyi +2 -2
  130. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  131. metaflow-stubs/runner/utils.pyi +4 -4
  132. metaflow-stubs/system/__init__.pyi +2 -2
  133. metaflow-stubs/system/system_logger.pyi +2 -2
  134. metaflow-stubs/system/system_monitor.pyi +2 -2
  135. metaflow-stubs/tagging_util.pyi +2 -2
  136. metaflow-stubs/tuple_util.pyi +2 -2
  137. metaflow-stubs/user_configs/__init__.pyi +2 -2
  138. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  139. metaflow-stubs/user_configs/config_options.pyi +2 -2
  140. metaflow-stubs/user_configs/config_parameters.pyi +7 -7
  141. metaflow-stubs/version.pyi +2 -2
  142. {metaflow_stubs-2.15.1.dist-info → metaflow_stubs-2.15.2.dist-info}/METADATA +2 -2
  143. metaflow_stubs-2.15.2.dist-info/RECORD +146 -0
  144. metaflow_stubs-2.15.1.dist-info/RECORD +0 -146
  145. {metaflow_stubs-2.15.1.dist-info → metaflow_stubs-2.15.2.dist-info}/WHEEL +0 -0
  146. {metaflow_stubs-2.15.1.dist-info → metaflow_stubs-2.15.2.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.1 #
4
- # Generated on 2025-02-28T22:52:51.105998 #
3
+ # MF version: 2.15.2 #
4
+ # Generated on 2025-03-01T03:37:26.010128 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -35,8 +35,8 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
- from . import tuple_util as tuple_util
39
38
  from . import events as events
39
+ from . import tuple_util as tuple_util
40
40
  from . import runner as runner
41
41
  from . import plugins as plugins
42
42
  from .plugins.datatools.s3.s3 import S3 as S3
@@ -145,41 +145,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
145
145
  """
146
146
  ...
147
147
 
148
- @typing.overload
149
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
150
- """
151
- Specifies secrets to be retrieved and injected as environment variables prior to
152
- the execution of a step.
153
-
154
-
155
- Parameters
156
- ----------
157
- sources : List[Union[str, Dict[str, Any]]], default: []
158
- List of secret specs, defining how the secrets are to be retrieved
159
- """
160
- ...
161
-
162
- @typing.overload
163
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
164
- ...
165
-
166
- @typing.overload
167
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
168
- ...
169
-
170
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
171
- """
172
- Specifies secrets to be retrieved and injected as environment variables prior to
173
- the execution of a step.
174
-
175
-
176
- Parameters
177
- ----------
178
- sources : List[Union[str, Dict[str, Any]]], default: []
179
- List of secret specs, defining how the secrets are to be retrieved
180
- """
181
- ...
182
-
183
148
  @typing.overload
184
149
  def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
185
150
  """
@@ -235,6 +200,127 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
235
200
  """
236
201
  ...
237
202
 
203
+ @typing.overload
204
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
205
+ """
206
+ Specifies that the step will success under all circumstances.
207
+
208
+ The decorator will create an optional artifact, specified by `var`, which
209
+ contains the exception raised. You can use it to detect the presence
210
+ of errors, indicating that all happy-path artifacts produced by the step
211
+ are missing.
212
+
213
+
214
+ Parameters
215
+ ----------
216
+ var : str, optional, default None
217
+ Name of the artifact in which to store the caught exception.
218
+ If not specified, the exception is not stored.
219
+ print_exception : bool, default True
220
+ Determines whether or not the exception is printed to
221
+ stdout when caught.
222
+ """
223
+ ...
224
+
225
+ @typing.overload
226
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
227
+ ...
228
+
229
+ @typing.overload
230
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
231
+ ...
232
+
233
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
234
+ """
235
+ Specifies that the step will success under all circumstances.
236
+
237
+ The decorator will create an optional artifact, specified by `var`, which
238
+ contains the exception raised. You can use it to detect the presence
239
+ of errors, indicating that all happy-path artifacts produced by the step
240
+ are missing.
241
+
242
+
243
+ Parameters
244
+ ----------
245
+ var : str, optional, default None
246
+ Name of the artifact in which to store the caught exception.
247
+ If not specified, the exception is not stored.
248
+ print_exception : bool, default True
249
+ Determines whether or not the exception is printed to
250
+ stdout when caught.
251
+ """
252
+ ...
253
+
254
+ @typing.overload
255
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
256
+ """
257
+ Decorator prototype for all step decorators. This function gets specialized
258
+ and imported for all decorators types by _import_plugin_decorators().
259
+ """
260
+ ...
261
+
262
+ @typing.overload
263
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
264
+ ...
265
+
266
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
267
+ """
268
+ Decorator prototype for all step decorators. This function gets specialized
269
+ and imported for all decorators types by _import_plugin_decorators().
270
+ """
271
+ ...
272
+
273
+ @typing.overload
274
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
275
+ """
276
+ Specifies the PyPI packages for the step.
277
+
278
+ Information in this decorator will augment any
279
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
280
+ you can use `@pypi_base` to set packages required by all
281
+ steps and use `@pypi` to specify step-specific overrides.
282
+
283
+
284
+ Parameters
285
+ ----------
286
+ packages : Dict[str, str], default: {}
287
+ Packages to use for this step. The key is the name of the package
288
+ and the value is the version to use.
289
+ python : str, optional, default: None
290
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
291
+ that the version used will correspond to the version of the Python interpreter used to start the run.
292
+ """
293
+ ...
294
+
295
+ @typing.overload
296
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
297
+ ...
298
+
299
+ @typing.overload
300
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
301
+ ...
302
+
303
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
304
+ """
305
+ Specifies the PyPI packages for the step.
306
+
307
+ Information in this decorator will augment any
308
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
309
+ you can use `@pypi_base` to set packages required by all
310
+ steps and use `@pypi` to specify step-specific overrides.
311
+
312
+
313
+ Parameters
314
+ ----------
315
+ packages : Dict[str, str], default: {}
316
+ Packages to use for this step. The key is the name of the package
317
+ and the value is the version to use.
318
+ python : str, optional, default: None
319
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
320
+ that the version used will correspond to the version of the Python interpreter used to start the run.
321
+ """
322
+ ...
323
+
238
324
  @typing.overload
239
325
  def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
240
326
  """
@@ -444,21 +530,96 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
444
530
  ...
445
531
 
446
532
  @typing.overload
447
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
533
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
448
534
  """
449
- Decorator prototype for all step decorators. This function gets specialized
450
- and imported for all decorators types by _import_plugin_decorators().
535
+ Specifies secrets to be retrieved and injected as environment variables prior to
536
+ the execution of a step.
537
+
538
+
539
+ Parameters
540
+ ----------
541
+ sources : List[Union[str, Dict[str, Any]]], default: []
542
+ List of secret specs, defining how the secrets are to be retrieved
451
543
  """
452
544
  ...
453
545
 
454
546
  @typing.overload
455
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
547
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
456
548
  ...
457
549
 
458
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
550
+ @typing.overload
551
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
552
+ ...
553
+
554
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
459
555
  """
460
- Decorator prototype for all step decorators. This function gets specialized
461
- and imported for all decorators types by _import_plugin_decorators().
556
+ Specifies secrets to be retrieved and injected as environment variables prior to
557
+ the execution of a step.
558
+
559
+
560
+ Parameters
561
+ ----------
562
+ sources : List[Union[str, Dict[str, Any]]], default: []
563
+ List of secret specs, defining how the secrets are to be retrieved
564
+ """
565
+ ...
566
+
567
+ @typing.overload
568
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
569
+ """
570
+ Specifies the Conda environment for the step.
571
+
572
+ Information in this decorator will augment any
573
+ attributes set in the `@conda_base` flow-level decorator. Hence,
574
+ you can use `@conda_base` to set packages required by all
575
+ steps and use `@conda` to specify step-specific overrides.
576
+
577
+
578
+ Parameters
579
+ ----------
580
+ packages : Dict[str, str], default {}
581
+ Packages to use for this step. The key is the name of the package
582
+ and the value is the version to use.
583
+ libraries : Dict[str, str], default {}
584
+ Supported for backward compatibility. When used with packages, packages will take precedence.
585
+ python : str, optional, default None
586
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
587
+ that the version used will correspond to the version of the Python interpreter used to start the run.
588
+ disabled : bool, default False
589
+ If set to True, disables @conda.
590
+ """
591
+ ...
592
+
593
+ @typing.overload
594
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
595
+ ...
596
+
597
+ @typing.overload
598
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
599
+ ...
600
+
601
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
602
+ """
603
+ Specifies the Conda environment for the step.
604
+
605
+ Information in this decorator will augment any
606
+ attributes set in the `@conda_base` flow-level decorator. Hence,
607
+ you can use `@conda_base` to set packages required by all
608
+ steps and use `@conda` to specify step-specific overrides.
609
+
610
+
611
+ Parameters
612
+ ----------
613
+ packages : Dict[str, str], default {}
614
+ Packages to use for this step. The key is the name of the package
615
+ and the value is the version to use.
616
+ libraries : Dict[str, str], default {}
617
+ Supported for backward compatibility. When used with packages, packages will take precedence.
618
+ python : str, optional, default None
619
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
620
+ that the version used will correspond to the version of the Python interpreter used to start the run.
621
+ disabled : bool, default False
622
+ If set to True, disables @conda.
462
623
  """
463
624
  ...
464
625
 
@@ -541,203 +702,9 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
541
702
  """
542
703
  ...
543
704
 
544
- @typing.overload
545
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
705
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
546
706
  """
547
- Specifies the PyPI packages for the step.
548
-
549
- Information in this decorator will augment any
550
- attributes set in the `@pyi_base` flow-level decorator. Hence,
551
- you can use `@pypi_base` to set packages required by all
552
- steps and use `@pypi` to specify step-specific overrides.
553
-
554
-
555
- Parameters
556
- ----------
557
- packages : Dict[str, str], default: {}
558
- Packages to use for this step. The key is the name of the package
559
- and the value is the version to use.
560
- python : str, optional, default: None
561
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
562
- that the version used will correspond to the version of the Python interpreter used to start the run.
563
- """
564
- ...
565
-
566
- @typing.overload
567
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
568
- ...
569
-
570
- @typing.overload
571
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
572
- ...
573
-
574
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
575
- """
576
- Specifies the PyPI packages for the step.
577
-
578
- Information in this decorator will augment any
579
- attributes set in the `@pyi_base` flow-level decorator. Hence,
580
- you can use `@pypi_base` to set packages required by all
581
- steps and use `@pypi` to specify step-specific overrides.
582
-
583
-
584
- Parameters
585
- ----------
586
- packages : Dict[str, str], default: {}
587
- Packages to use for this step. The key is the name of the package
588
- and the value is the version to use.
589
- python : str, optional, default: None
590
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
591
- that the version used will correspond to the version of the Python interpreter used to start the run.
592
- """
593
- ...
594
-
595
- @typing.overload
596
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
597
- """
598
- Specifies that the step will success under all circumstances.
599
-
600
- The decorator will create an optional artifact, specified by `var`, which
601
- contains the exception raised. You can use it to detect the presence
602
- of errors, indicating that all happy-path artifacts produced by the step
603
- are missing.
604
-
605
-
606
- Parameters
607
- ----------
608
- var : str, optional, default None
609
- Name of the artifact in which to store the caught exception.
610
- If not specified, the exception is not stored.
611
- print_exception : bool, default True
612
- Determines whether or not the exception is printed to
613
- stdout when caught.
614
- """
615
- ...
616
-
617
- @typing.overload
618
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
619
- ...
620
-
621
- @typing.overload
622
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
623
- ...
624
-
625
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
626
- """
627
- Specifies that the step will success under all circumstances.
628
-
629
- The decorator will create an optional artifact, specified by `var`, which
630
- contains the exception raised. You can use it to detect the presence
631
- of errors, indicating that all happy-path artifacts produced by the step
632
- are missing.
633
-
634
-
635
- Parameters
636
- ----------
637
- var : str, optional, default None
638
- Name of the artifact in which to store the caught exception.
639
- If not specified, the exception is not stored.
640
- print_exception : bool, default True
641
- Determines whether or not the exception is printed to
642
- stdout when caught.
643
- """
644
- ...
645
-
646
- @typing.overload
647
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
648
- """
649
- Specifies the Conda environment for the step.
650
-
651
- Information in this decorator will augment any
652
- attributes set in the `@conda_base` flow-level decorator. Hence,
653
- you can use `@conda_base` to set packages required by all
654
- steps and use `@conda` to specify step-specific overrides.
655
-
656
-
657
- Parameters
658
- ----------
659
- packages : Dict[str, str], default {}
660
- Packages to use for this step. The key is the name of the package
661
- and the value is the version to use.
662
- libraries : Dict[str, str], default {}
663
- Supported for backward compatibility. When used with packages, packages will take precedence.
664
- python : str, optional, default None
665
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
666
- that the version used will correspond to the version of the Python interpreter used to start the run.
667
- disabled : bool, default False
668
- If set to True, disables @conda.
669
- """
670
- ...
671
-
672
- @typing.overload
673
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
674
- ...
675
-
676
- @typing.overload
677
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
678
- ...
679
-
680
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
681
- """
682
- Specifies the Conda environment for the step.
683
-
684
- Information in this decorator will augment any
685
- attributes set in the `@conda_base` flow-level decorator. Hence,
686
- you can use `@conda_base` to set packages required by all
687
- steps and use `@conda` to specify step-specific overrides.
688
-
689
-
690
- Parameters
691
- ----------
692
- packages : Dict[str, str], default {}
693
- Packages to use for this step. The key is the name of the package
694
- and the value is the version to use.
695
- libraries : Dict[str, str], default {}
696
- Supported for backward compatibility. When used with packages, packages will take precedence.
697
- python : str, optional, default None
698
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
699
- that the version used will correspond to the version of the Python interpreter used to start the run.
700
- disabled : bool, default False
701
- If set to True, disables @conda.
702
- """
703
- ...
704
-
705
- @typing.overload
706
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
707
- """
708
- Specifies environment variables to be set prior to the execution of a step.
709
-
710
-
711
- Parameters
712
- ----------
713
- vars : Dict[str, str], default {}
714
- Dictionary of environment variables to set.
715
- """
716
- ...
717
-
718
- @typing.overload
719
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
720
- ...
721
-
722
- @typing.overload
723
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
724
- ...
725
-
726
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
727
- """
728
- Specifies environment variables to be set prior to the execution of a step.
729
-
730
-
731
- Parameters
732
- ----------
733
- vars : Dict[str, str], default {}
734
- Dictionary of environment variables to set.
735
- """
736
- ...
737
-
738
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
739
- """
740
- Specifies that this step should execute on Kubernetes.
707
+ Specifies that this step should execute on Kubernetes.
741
708
 
742
709
 
743
710
  Parameters
@@ -861,87 +828,215 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
861
828
  """
862
829
  ...
863
830
 
864
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
831
+ @typing.overload
832
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
865
833
  """
866
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
867
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
834
+ Specifies environment variables to be set prior to the execution of a step.
868
835
 
869
836
 
870
837
  Parameters
871
838
  ----------
872
- timeout : int
873
- Time, in seconds before the task times out and fails. (Default: 3600)
874
- poke_interval : int
875
- Time in seconds that the job should wait in between each try. (Default: 60)
876
- mode : str
877
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
878
- exponential_backoff : bool
879
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
880
- pool : str
881
- the slot pool this task should run in,
882
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
883
- soft_fail : bool
884
- Set to true to mark the task as SKIPPED on failure. (Default: False)
885
- name : str
886
- Name of the sensor on Airflow
887
- description : str
888
- Description of sensor in the Airflow UI
889
- external_dag_id : str
890
- The dag_id that contains the task you want to wait for.
891
- external_task_ids : List[str]
892
- The list of task_ids that you want to wait for.
893
- If None (default value) the sensor waits for the DAG. (Default: None)
894
- allowed_states : List[str]
895
- Iterable of allowed states, (Default: ['success'])
896
- failed_states : List[str]
897
- Iterable of failed or dis-allowed states. (Default: None)
898
- execution_delta : datetime.timedelta
899
- time difference with the previous execution to look at,
900
- the default is the same logical date as the current task or DAG. (Default: None)
901
- check_existence: bool
902
- Set to True to check if the external task exists or check if
903
- the DAG to wait for exists. (Default: True)
839
+ vars : Dict[str, str], default {}
840
+ Dictionary of environment variables to set.
904
841
  """
905
842
  ...
906
843
 
907
844
  @typing.overload
908
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
845
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
846
+ ...
847
+
848
+ @typing.overload
849
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
850
+ ...
851
+
852
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
909
853
  """
910
- Specifies the PyPI packages for all steps of the flow.
854
+ Specifies environment variables to be set prior to the execution of a step.
911
855
 
912
- Use `@pypi_base` to set common packages required by all
913
- steps and use `@pypi` to specify step-specific overrides.
914
856
 
915
857
  Parameters
916
858
  ----------
917
- packages : Dict[str, str], default: {}
918
- Packages to use for this flow. The key is the name of the package
919
- and the value is the version to use.
920
- python : str, optional, default: None
921
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
922
- that the version used will correspond to the version of the Python interpreter used to start the run.
859
+ vars : Dict[str, str], default {}
860
+ Dictionary of environment variables to set.
861
+ """
862
+ ...
863
+
864
+ @typing.overload
865
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
866
+ """
867
+ Specifies the times when the flow should be run when running on a
868
+ production scheduler.
869
+
870
+
871
+ Parameters
872
+ ----------
873
+ hourly : bool, default False
874
+ Run the workflow hourly.
875
+ daily : bool, default True
876
+ Run the workflow daily.
877
+ weekly : bool, default False
878
+ Run the workflow weekly.
879
+ cron : str, optional, default None
880
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
881
+ specified by this expression.
882
+ timezone : str, optional, default None
883
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
884
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
885
+ """
886
+ ...
887
+
888
+ @typing.overload
889
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
890
+ ...
891
+
892
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
893
+ """
894
+ Specifies the times when the flow should be run when running on a
895
+ production scheduler.
896
+
897
+
898
+ Parameters
899
+ ----------
900
+ hourly : bool, default False
901
+ Run the workflow hourly.
902
+ daily : bool, default True
903
+ Run the workflow daily.
904
+ weekly : bool, default False
905
+ Run the workflow weekly.
906
+ cron : str, optional, default None
907
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
908
+ specified by this expression.
909
+ timezone : str, optional, default None
910
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
911
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
912
+ """
913
+ ...
914
+
915
+ @typing.overload
916
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
917
+ """
918
+ Specifies the event(s) that this flow depends on.
919
+
920
+ ```
921
+ @trigger(event='foo')
922
+ ```
923
+ or
924
+ ```
925
+ @trigger(events=['foo', 'bar'])
926
+ ```
927
+
928
+ Additionally, you can specify the parameter mappings
929
+ to map event payload to Metaflow parameters for the flow.
930
+ ```
931
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
932
+ ```
933
+ or
934
+ ```
935
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
936
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
937
+ ```
938
+
939
+ 'parameters' can also be a list of strings and tuples like so:
940
+ ```
941
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
942
+ ```
943
+ This is equivalent to:
944
+ ```
945
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
946
+ ```
947
+
948
+
949
+ Parameters
950
+ ----------
951
+ event : Union[str, Dict[str, Any]], optional, default None
952
+ Event dependency for this flow.
953
+ events : List[Union[str, Dict[str, Any]]], default []
954
+ Events dependency for this flow.
955
+ options : Dict[str, Any], default {}
956
+ Backend-specific configuration for tuning eventing behavior.
957
+ """
958
+ ...
959
+
960
+ @typing.overload
961
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
962
+ ...
963
+
964
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
965
+ """
966
+ Specifies the event(s) that this flow depends on.
967
+
968
+ ```
969
+ @trigger(event='foo')
970
+ ```
971
+ or
972
+ ```
973
+ @trigger(events=['foo', 'bar'])
974
+ ```
975
+
976
+ Additionally, you can specify the parameter mappings
977
+ to map event payload to Metaflow parameters for the flow.
978
+ ```
979
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
980
+ ```
981
+ or
982
+ ```
983
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
984
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
985
+ ```
986
+
987
+ 'parameters' can also be a list of strings and tuples like so:
988
+ ```
989
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
990
+ ```
991
+ This is equivalent to:
992
+ ```
993
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
994
+ ```
995
+
996
+
997
+ Parameters
998
+ ----------
999
+ event : Union[str, Dict[str, Any]], optional, default None
1000
+ Event dependency for this flow.
1001
+ events : List[Union[str, Dict[str, Any]]], default []
1002
+ Events dependency for this flow.
1003
+ options : Dict[str, Any], default {}
1004
+ Backend-specific configuration for tuning eventing behavior.
923
1005
  """
924
1006
  ...
925
1007
 
926
- @typing.overload
927
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
928
- ...
929
-
930
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1008
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
931
1009
  """
932
- Specifies the PyPI packages for all steps of the flow.
1010
+ Specifies what flows belong to the same project.
1011
+
1012
+ A project-specific namespace is created for all flows that
1013
+ use the same `@project(name)`.
933
1014
 
934
- Use `@pypi_base` to set common packages required by all
935
- steps and use `@pypi` to specify step-specific overrides.
936
1015
 
937
1016
  Parameters
938
1017
  ----------
939
- packages : Dict[str, str], default: {}
940
- Packages to use for this flow. The key is the name of the package
941
- and the value is the version to use.
942
- python : str, optional, default: None
943
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
944
- that the version used will correspond to the version of the Python interpreter used to start the run.
1018
+ name : str
1019
+ Project name. Make sure that the name is unique amongst all
1020
+ projects that use the same production scheduler. The name may
1021
+ contain only lowercase alphanumeric characters and underscores.
1022
+
1023
+ branch : Optional[str], default None
1024
+ The branch to use. If not specified, the branch is set to
1025
+ `user.<username>` unless `production` is set to `True`. This can
1026
+ also be set on the command line using `--branch` as a top-level option.
1027
+ It is an error to specify `branch` in the decorator and on the command line.
1028
+
1029
+ production : bool, default False
1030
+ Whether or not the branch is the production branch. This can also be set on the
1031
+ command line using `--production` as a top-level option. It is an error to specify
1032
+ `production` in the decorator and on the command line.
1033
+ The project branch name will be:
1034
+ - if `branch` is specified:
1035
+ - if `production` is True: `prod.<branch>`
1036
+ - if `production` is False: `test.<branch>`
1037
+ - if `branch` is not specified:
1038
+ - if `production` is True: `prod`
1039
+ - if `production` is False: `user.<username>`
945
1040
  """
946
1041
  ...
947
1042
 
@@ -988,54 +1083,46 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
988
1083
  """
989
1084
  ...
990
1085
 
991
- @typing.overload
992
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
993
- """
994
- Specifies the times when the flow should be run when running on a
995
- production scheduler.
996
-
997
-
998
- Parameters
999
- ----------
1000
- hourly : bool, default False
1001
- Run the workflow hourly.
1002
- daily : bool, default True
1003
- Run the workflow daily.
1004
- weekly : bool, default False
1005
- Run the workflow weekly.
1006
- cron : str, optional, default None
1007
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1008
- specified by this expression.
1009
- timezone : str, optional, default None
1010
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1011
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1012
- """
1013
- ...
1014
-
1015
- @typing.overload
1016
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1017
- ...
1018
-
1019
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1086
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1020
1087
  """
1021
- Specifies the times when the flow should be run when running on a
1022
- production scheduler.
1088
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1089
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1023
1090
 
1024
1091
 
1025
1092
  Parameters
1026
1093
  ----------
1027
- hourly : bool, default False
1028
- Run the workflow hourly.
1029
- daily : bool, default True
1030
- Run the workflow daily.
1031
- weekly : bool, default False
1032
- Run the workflow weekly.
1033
- cron : str, optional, default None
1034
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1035
- specified by this expression.
1036
- timezone : str, optional, default None
1037
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1038
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1094
+ timeout : int
1095
+ Time, in seconds before the task times out and fails. (Default: 3600)
1096
+ poke_interval : int
1097
+ Time in seconds that the job should wait in between each try. (Default: 60)
1098
+ mode : str
1099
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1100
+ exponential_backoff : bool
1101
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1102
+ pool : str
1103
+ the slot pool this task should run in,
1104
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1105
+ soft_fail : bool
1106
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1107
+ name : str
1108
+ Name of the sensor on Airflow
1109
+ description : str
1110
+ Description of sensor in the Airflow UI
1111
+ external_dag_id : str
1112
+ The dag_id that contains the task you want to wait for.
1113
+ external_task_ids : List[str]
1114
+ The list of task_ids that you want to wait for.
1115
+ If None (default value) the sensor waits for the DAG. (Default: None)
1116
+ allowed_states : List[str]
1117
+ Iterable of allowed states, (Default: ['success'])
1118
+ failed_states : List[str]
1119
+ Iterable of failed or dis-allowed states. (Default: None)
1120
+ execution_delta : datetime.timedelta
1121
+ time difference with the previous execution to look at,
1122
+ the default is the same logical date as the current task or DAG. (Default: None)
1123
+ check_existence: bool
1124
+ Set to True to check if the external task exists or check if
1125
+ the DAG to wait for exists. (Default: True)
1039
1126
  """
1040
1127
  ...
1041
1128
 
@@ -1090,41 +1177,6 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1090
1177
  """
1091
1178
  ...
1092
1179
 
1093
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1094
- """
1095
- Specifies what flows belong to the same project.
1096
-
1097
- A project-specific namespace is created for all flows that
1098
- use the same `@project(name)`.
1099
-
1100
-
1101
- Parameters
1102
- ----------
1103
- name : str
1104
- Project name. Make sure that the name is unique amongst all
1105
- projects that use the same production scheduler. The name may
1106
- contain only lowercase alphanumeric characters and underscores.
1107
-
1108
- branch : Optional[str], default None
1109
- The branch to use. If not specified, the branch is set to
1110
- `user.<username>` unless `production` is set to `True`. This can
1111
- also be set on the command line using `--branch` as a top-level option.
1112
- It is an error to specify `branch` in the decorator and on the command line.
1113
-
1114
- production : bool, default False
1115
- Whether or not the branch is the production branch. This can also be set on the
1116
- command line using `--production` as a top-level option. It is an error to specify
1117
- `production` in the decorator and on the command line.
1118
- The project branch name will be:
1119
- - if `branch` is specified:
1120
- - if `production` is True: `prod.<branch>`
1121
- - if `production` is False: `test.<branch>`
1122
- - if `branch` is not specified:
1123
- - if `production` is True: `prod`
1124
- - if `production` is False: `user.<username>`
1125
- """
1126
- ...
1127
-
1128
1180
  @typing.overload
1129
1181
  def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1130
1182
  """
@@ -1227,95 +1279,43 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1227
1279
  ...
1228
1280
 
1229
1281
  @typing.overload
1230
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1282
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1231
1283
  """
1232
- Specifies the event(s) that this flow depends on.
1233
-
1234
- ```
1235
- @trigger(event='foo')
1236
- ```
1237
- or
1238
- ```
1239
- @trigger(events=['foo', 'bar'])
1240
- ```
1241
-
1242
- Additionally, you can specify the parameter mappings
1243
- to map event payload to Metaflow parameters for the flow.
1244
- ```
1245
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1246
- ```
1247
- or
1248
- ```
1249
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1250
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1251
- ```
1252
-
1253
- 'parameters' can also be a list of strings and tuples like so:
1254
- ```
1255
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1256
- ```
1257
- This is equivalent to:
1258
- ```
1259
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1260
- ```
1284
+ Specifies the PyPI packages for all steps of the flow.
1261
1285
 
1286
+ Use `@pypi_base` to set common packages required by all
1287
+ steps and use `@pypi` to specify step-specific overrides.
1262
1288
 
1263
1289
  Parameters
1264
1290
  ----------
1265
- event : Union[str, Dict[str, Any]], optional, default None
1266
- Event dependency for this flow.
1267
- events : List[Union[str, Dict[str, Any]]], default []
1268
- Events dependency for this flow.
1269
- options : Dict[str, Any], default {}
1270
- Backend-specific configuration for tuning eventing behavior.
1291
+ packages : Dict[str, str], default: {}
1292
+ Packages to use for this flow. The key is the name of the package
1293
+ and the value is the version to use.
1294
+ python : str, optional, default: None
1295
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1296
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1271
1297
  """
1272
1298
  ...
1273
1299
 
1274
1300
  @typing.overload
1275
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1301
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1276
1302
  ...
1277
1303
 
1278
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1304
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1279
1305
  """
1280
- Specifies the event(s) that this flow depends on.
1281
-
1282
- ```
1283
- @trigger(event='foo')
1284
- ```
1285
- or
1286
- ```
1287
- @trigger(events=['foo', 'bar'])
1288
- ```
1289
-
1290
- Additionally, you can specify the parameter mappings
1291
- to map event payload to Metaflow parameters for the flow.
1292
- ```
1293
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1294
- ```
1295
- or
1296
- ```
1297
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1298
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1299
- ```
1300
-
1301
- 'parameters' can also be a list of strings and tuples like so:
1302
- ```
1303
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1304
- ```
1305
- This is equivalent to:
1306
- ```
1307
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1308
- ```
1306
+ Specifies the PyPI packages for all steps of the flow.
1309
1307
 
1308
+ Use `@pypi_base` to set common packages required by all
1309
+ steps and use `@pypi` to specify step-specific overrides.
1310
1310
 
1311
1311
  Parameters
1312
1312
  ----------
1313
- event : Union[str, Dict[str, Any]], optional, default None
1314
- Event dependency for this flow.
1315
- events : List[Union[str, Dict[str, Any]]], default []
1316
- Events dependency for this flow.
1317
- options : Dict[str, Any], default {}
1318
- Backend-specific configuration for tuning eventing behavior.
1313
+ packages : Dict[str, str], default: {}
1314
+ Packages to use for this flow. The key is the name of the package
1315
+ and the value is the version to use.
1316
+ python : str, optional, default: None
1317
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1318
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1319
1319
  """
1320
1320
  ...
1321
1321