metaflow-stubs 2.16.1__py2.py3-none-any.whl → 2.16.3__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of metaflow-stubs might be problematic. Click here for more details.

Files changed (166) hide show
  1. metaflow-stubs/__init__.pyi +637 -637
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +23 -23
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/multicore_utils.pyi +2 -2
  23. metaflow-stubs/packaging_sys/__init__.pyi +6 -6
  24. metaflow-stubs/packaging_sys/backend.pyi +4 -4
  25. metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
  26. metaflow-stubs/packaging_sys/tar_backend.pyi +7 -7
  27. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  28. metaflow-stubs/packaging_sys/v1.pyi +2 -2
  29. metaflow-stubs/parameters.pyi +3 -3
  30. metaflow-stubs/plugins/__init__.pyi +11 -11
  31. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  33. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  34. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  35. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  37. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  38. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  39. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  41. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  42. metaflow-stubs/plugins/argo/argo_workflows.pyi +5 -5
  43. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  44. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  45. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  46. metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
  47. metaflow-stubs/plugins/aws/__init__.pyi +3 -3
  48. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  50. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  51. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  52. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  53. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  54. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  55. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  56. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  58. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  59. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  60. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  61. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  62. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  63. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  64. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  65. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  67. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  68. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  69. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  70. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  71. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  72. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  81. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  84. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  85. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  86. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  87. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  92. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  93. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  94. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  95. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  96. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  104. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  105. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  106. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  107. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  108. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  110. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  112. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  114. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  115. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  116. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  119. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  121. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  122. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  123. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  124. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  125. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  126. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  127. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  128. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  129. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  130. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  131. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  132. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  133. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  134. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  135. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  136. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  138. metaflow-stubs/pylint_wrapper.pyi +2 -2
  139. metaflow-stubs/runner/__init__.pyi +2 -2
  140. metaflow-stubs/runner/deployer.pyi +30 -30
  141. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  142. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  143. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  144. metaflow-stubs/runner/nbrun.pyi +2 -2
  145. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  146. metaflow-stubs/runner/utils.pyi +3 -3
  147. metaflow-stubs/system/__init__.pyi +2 -2
  148. metaflow-stubs/system/system_logger.pyi +3 -3
  149. metaflow-stubs/system/system_monitor.pyi +2 -2
  150. metaflow-stubs/tagging_util.pyi +2 -2
  151. metaflow-stubs/tuple_util.pyi +2 -2
  152. metaflow-stubs/user_configs/__init__.pyi +2 -2
  153. metaflow-stubs/user_configs/config_options.pyi +2 -2
  154. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  155. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  156. metaflow-stubs/user_decorators/common.pyi +2 -2
  157. metaflow-stubs/user_decorators/mutable_flow.pyi +6 -6
  158. metaflow-stubs/user_decorators/mutable_step.pyi +6 -6
  159. metaflow-stubs/user_decorators/user_flow_decorator.pyi +3 -3
  160. metaflow-stubs/user_decorators/user_step_decorator.pyi +5 -5
  161. metaflow-stubs/version.pyi +2 -2
  162. {metaflow_stubs-2.16.1.dist-info → metaflow_stubs-2.16.3.dist-info}/METADATA +2 -2
  163. metaflow_stubs-2.16.3.dist-info/RECORD +166 -0
  164. metaflow_stubs-2.16.1.dist-info/RECORD +0 -166
  165. {metaflow_stubs-2.16.1.dist-info → metaflow_stubs-2.16.3.dist-info}/WHEEL +0 -0
  166. {metaflow_stubs-2.16.1.dist-info → metaflow_stubs-2.16.3.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.16.1 #
4
- # Generated on 2025-07-15T19:29:43.053659 #
3
+ # MF version: 2.16.3 #
4
+ # Generated on 2025-07-16T15:23:11.248549 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -39,17 +39,17 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
39
39
  from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
+ from . import tuple_util as tuple_util
42
43
  from . import metaflow_git as metaflow_git
43
44
  from . import events as events
44
- from . import tuple_util as tuple_util
45
45
  from . import runner as runner
46
46
  from . import plugins as plugins
47
47
  from .plugins.datatools.s3.s3 import S3 as S3
48
48
  from . import includefile as includefile
49
49
  from .includefile import IncludeFile as IncludeFile
50
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
51
50
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
52
51
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
52
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
53
53
  from . import cards as cards
54
54
  from . import client as client
55
55
  from .client.core import namespace as namespace
@@ -153,302 +153,426 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
153
153
  ...
154
154
 
155
155
  @typing.overload
156
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
156
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
157
157
  """
158
- Specifies the Conda environment for the step.
159
-
160
- Information in this decorator will augment any
161
- attributes set in the `@conda_base` flow-level decorator. Hence,
162
- you can use `@conda_base` to set packages required by all
163
- steps and use `@conda` to specify step-specific overrides.
158
+ Specifies environment variables to be set prior to the execution of a step.
164
159
 
165
160
 
166
161
  Parameters
167
162
  ----------
168
- packages : Dict[str, str], default {}
169
- Packages to use for this step. The key is the name of the package
170
- and the value is the version to use.
171
- libraries : Dict[str, str], default {}
172
- Supported for backward compatibility. When used with packages, packages will take precedence.
173
- python : str, optional, default None
174
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
175
- that the version used will correspond to the version of the Python interpreter used to start the run.
176
- disabled : bool, default False
177
- If set to True, disables @conda.
163
+ vars : Dict[str, str], default {}
164
+ Dictionary of environment variables to set.
178
165
  """
179
166
  ...
180
167
 
181
168
  @typing.overload
182
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
169
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
183
170
  ...
184
171
 
185
172
  @typing.overload
186
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
173
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
187
174
  ...
188
175
 
189
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
176
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
190
177
  """
191
- Specifies the Conda environment for the step.
192
-
193
- Information in this decorator will augment any
194
- attributes set in the `@conda_base` flow-level decorator. Hence,
195
- you can use `@conda_base` to set packages required by all
196
- steps and use `@conda` to specify step-specific overrides.
178
+ Specifies environment variables to be set prior to the execution of a step.
197
179
 
198
180
 
199
181
  Parameters
200
182
  ----------
201
- packages : Dict[str, str], default {}
202
- Packages to use for this step. The key is the name of the package
203
- and the value is the version to use.
204
- libraries : Dict[str, str], default {}
205
- Supported for backward compatibility. When used with packages, packages will take precedence.
206
- python : str, optional, default None
207
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
208
- that the version used will correspond to the version of the Python interpreter used to start the run.
209
- disabled : bool, default False
210
- If set to True, disables @conda.
183
+ vars : Dict[str, str], default {}
184
+ Dictionary of environment variables to set.
211
185
  """
212
186
  ...
213
187
 
214
188
  @typing.overload
215
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
189
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
216
190
  """
217
- Specifies a timeout for your step.
218
-
219
- This decorator is useful if this step may hang indefinitely.
220
-
221
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
222
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
223
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
191
+ Creates a human-readable report, a Metaflow Card, after this step completes.
224
192
 
225
- Note that all the values specified in parameters are added together so if you specify
226
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
193
+ Note that you may add multiple `@card` decorators in a step with different parameters.
227
194
 
228
195
 
229
196
  Parameters
230
197
  ----------
231
- seconds : int, default 0
232
- Number of seconds to wait prior to timing out.
233
- minutes : int, default 0
234
- Number of minutes to wait prior to timing out.
235
- hours : int, default 0
236
- Number of hours to wait prior to timing out.
198
+ type : str, default 'default'
199
+ Card type.
200
+ id : str, optional, default None
201
+ If multiple cards are present, use this id to identify this card.
202
+ options : Dict[str, Any], default {}
203
+ Options passed to the card. The contents depend on the card type.
204
+ timeout : int, default 45
205
+ Interrupt reporting if it takes more than this many seconds.
237
206
  """
238
207
  ...
239
208
 
240
209
  @typing.overload
241
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
210
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
242
211
  ...
243
212
 
244
213
  @typing.overload
245
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
214
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
246
215
  ...
247
216
 
248
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
217
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
249
218
  """
250
- Specifies a timeout for your step.
251
-
252
- This decorator is useful if this step may hang indefinitely.
253
-
254
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
255
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
256
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
219
+ Creates a human-readable report, a Metaflow Card, after this step completes.
257
220
 
258
- Note that all the values specified in parameters are added together so if you specify
259
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
221
+ Note that you may add multiple `@card` decorators in a step with different parameters.
260
222
 
261
223
 
262
224
  Parameters
263
225
  ----------
264
- seconds : int, default 0
265
- Number of seconds to wait prior to timing out.
266
- minutes : int, default 0
267
- Number of minutes to wait prior to timing out.
268
- hours : int, default 0
269
- Number of hours to wait prior to timing out.
226
+ type : str, default 'default'
227
+ Card type.
228
+ id : str, optional, default None
229
+ If multiple cards are present, use this id to identify this card.
230
+ options : Dict[str, Any], default {}
231
+ Options passed to the card. The contents depend on the card type.
232
+ timeout : int, default 45
233
+ Interrupt reporting if it takes more than this many seconds.
270
234
  """
271
235
  ...
272
236
 
273
237
  @typing.overload
274
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
238
+ def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
275
239
  """
276
- Specifies the resources needed when executing this step.
277
-
278
- Use `@resources` to specify the resource requirements
279
- independently of the specific compute layer (`@batch`, `@kubernetes`).
280
-
281
- You can choose the compute layer on the command line by executing e.g.
282
- ```
283
- python myflow.py run --with batch
284
- ```
285
- or
286
- ```
287
- python myflow.py run --with kubernetes
288
- ```
289
- which executes the flow on the desired system using the
290
- requirements specified in `@resources`.
240
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
291
241
 
292
242
 
293
243
  Parameters
294
244
  ----------
295
245
  cpu : int, default 1
296
- Number of CPUs required for this step.
297
- gpu : int, optional, default None
298
- Number of GPUs required for this step.
299
- disk : int, optional, default None
300
- Disk size (in MB) required for this step. Only applies on Kubernetes.
246
+ Number of CPUs required for this step. If `@resources` is
247
+ also present, the maximum value from all decorators is used.
248
+ gpu : int, default 0
249
+ Number of GPUs required for this step. If `@resources` is
250
+ also present, the maximum value from all decorators is used.
301
251
  memory : int, default 4096
302
- Memory size (in MB) required for this step.
252
+ Memory size (in MB) required for this step. If
253
+ `@resources` is also present, the maximum value from all decorators is
254
+ used.
255
+ image : str, optional, default None
256
+ Docker image to use when launching on AWS Batch. If not specified, and
257
+ METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
258
+ not, a default Docker image mapping to the current version of Python is used.
259
+ queue : str, default METAFLOW_BATCH_JOB_QUEUE
260
+ AWS Batch Job Queue to submit the job to.
261
+ iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
262
+ AWS IAM role that AWS Batch container uses to access AWS cloud resources.
263
+ execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
264
+ AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
265
+ (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
303
266
  shared_memory : int, optional, default None
304
267
  The value for the size (in MiB) of the /dev/shm volume for this step.
305
268
  This parameter maps to the `--shm-size` option in Docker.
269
+ max_swap : int, optional, default None
270
+ The total amount of swap memory (in MiB) a container can use for this
271
+ step. This parameter is translated to the `--memory-swap` option in
272
+ Docker where the value is the sum of the container memory plus the
273
+ `max_swap` value.
274
+ swappiness : int, optional, default None
275
+ This allows you to tune memory swappiness behavior for this step.
276
+ A swappiness value of 0 causes swapping not to happen unless absolutely
277
+ necessary. A swappiness value of 100 causes pages to be swapped very
278
+ aggressively. Accepted values are whole numbers between 0 and 100.
279
+ use_tmpfs : bool, default False
280
+ This enables an explicit tmpfs mount for this step. Note that tmpfs is
281
+ not available on Fargate compute environments
282
+ tmpfs_tempdir : bool, default True
283
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
284
+ tmpfs_size : int, optional, default None
285
+ The value for the size (in MiB) of the tmpfs mount for this step.
286
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
287
+ memory allocated for this step.
288
+ tmpfs_path : str, optional, default None
289
+ Path to tmpfs mount for this step. Defaults to /metaflow_temp.
290
+ inferentia : int, default 0
291
+ Number of Inferentia chips required for this step.
292
+ trainium : int, default None
293
+ Alias for inferentia. Use only one of the two.
294
+ efa : int, default 0
295
+ Number of elastic fabric adapter network devices to attach to container
296
+ ephemeral_storage : int, default None
297
+ The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
298
+ This is only relevant for Fargate compute environments
299
+ log_driver: str, optional, default None
300
+ The log driver to use for the Amazon ECS container.
301
+ log_options: List[str], optional, default None
302
+ List of strings containing options for the chosen log driver. The configurable values
303
+ depend on the `log driver` chosen. Validation of these options is not supported yet.
304
+ Example: [`awslogs-group:aws/batch/job`]
306
305
  """
307
306
  ...
308
307
 
309
308
  @typing.overload
310
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
309
+ def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
311
310
  ...
312
311
 
313
312
  @typing.overload
314
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
313
+ def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
315
314
  ...
316
315
 
317
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
316
+ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
318
317
  """
319
- Specifies the resources needed when executing this step.
320
-
321
- Use `@resources` to specify the resource requirements
322
- independently of the specific compute layer (`@batch`, `@kubernetes`).
323
-
324
- You can choose the compute layer on the command line by executing e.g.
325
- ```
326
- python myflow.py run --with batch
327
- ```
328
- or
329
- ```
330
- python myflow.py run --with kubernetes
331
- ```
332
- which executes the flow on the desired system using the
333
- requirements specified in `@resources`.
318
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
334
319
 
335
320
 
336
321
  Parameters
337
322
  ----------
338
323
  cpu : int, default 1
339
- Number of CPUs required for this step.
340
- gpu : int, optional, default None
341
- Number of GPUs required for this step.
342
- disk : int, optional, default None
343
- Disk size (in MB) required for this step. Only applies on Kubernetes.
324
+ Number of CPUs required for this step. If `@resources` is
325
+ also present, the maximum value from all decorators is used.
326
+ gpu : int, default 0
327
+ Number of GPUs required for this step. If `@resources` is
328
+ also present, the maximum value from all decorators is used.
344
329
  memory : int, default 4096
345
- Memory size (in MB) required for this step.
330
+ Memory size (in MB) required for this step. If
331
+ `@resources` is also present, the maximum value from all decorators is
332
+ used.
333
+ image : str, optional, default None
334
+ Docker image to use when launching on AWS Batch. If not specified, and
335
+ METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
336
+ not, a default Docker image mapping to the current version of Python is used.
337
+ queue : str, default METAFLOW_BATCH_JOB_QUEUE
338
+ AWS Batch Job Queue to submit the job to.
339
+ iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
340
+ AWS IAM role that AWS Batch container uses to access AWS cloud resources.
341
+ execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
342
+ AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
343
+ (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
346
344
  shared_memory : int, optional, default None
347
345
  The value for the size (in MiB) of the /dev/shm volume for this step.
348
346
  This parameter maps to the `--shm-size` option in Docker.
349
- """
350
- ...
351
-
352
- @typing.overload
353
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
354
- """
355
- Specifies environment variables to be set prior to the execution of a step.
356
-
357
-
358
- Parameters
359
- ----------
360
- vars : Dict[str, str], default {}
361
- Dictionary of environment variables to set.
362
- """
363
- ...
364
-
365
- @typing.overload
366
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
367
- ...
368
-
369
- @typing.overload
370
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
347
+ max_swap : int, optional, default None
348
+ The total amount of swap memory (in MiB) a container can use for this
349
+ step. This parameter is translated to the `--memory-swap` option in
350
+ Docker where the value is the sum of the container memory plus the
351
+ `max_swap` value.
352
+ swappiness : int, optional, default None
353
+ This allows you to tune memory swappiness behavior for this step.
354
+ A swappiness value of 0 causes swapping not to happen unless absolutely
355
+ necessary. A swappiness value of 100 causes pages to be swapped very
356
+ aggressively. Accepted values are whole numbers between 0 and 100.
357
+ use_tmpfs : bool, default False
358
+ This enables an explicit tmpfs mount for this step. Note that tmpfs is
359
+ not available on Fargate compute environments
360
+ tmpfs_tempdir : bool, default True
361
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
362
+ tmpfs_size : int, optional, default None
363
+ The value for the size (in MiB) of the tmpfs mount for this step.
364
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
365
+ memory allocated for this step.
366
+ tmpfs_path : str, optional, default None
367
+ Path to tmpfs mount for this step. Defaults to /metaflow_temp.
368
+ inferentia : int, default 0
369
+ Number of Inferentia chips required for this step.
370
+ trainium : int, default None
371
+ Alias for inferentia. Use only one of the two.
372
+ efa : int, default 0
373
+ Number of elastic fabric adapter network devices to attach to container
374
+ ephemeral_storage : int, default None
375
+ The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
376
+ This is only relevant for Fargate compute environments
377
+ log_driver: str, optional, default None
378
+ The log driver to use for the Amazon ECS container.
379
+ log_options: List[str], optional, default None
380
+ List of strings containing options for the chosen log driver. The configurable values
381
+ depend on the `log driver` chosen. Validation of these options is not supported yet.
382
+ Example: [`awslogs-group:aws/batch/job`]
383
+ """
371
384
  ...
372
385
 
373
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
386
+ @typing.overload
387
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
374
388
  """
375
- Specifies environment variables to be set prior to the execution of a step.
389
+ Specifies the number of times the task corresponding
390
+ to a step needs to be retried.
391
+
392
+ This decorator is useful for handling transient errors, such as networking issues.
393
+ If your task contains operations that can't be retried safely, e.g. database updates,
394
+ it is advisable to annotate it with `@retry(times=0)`.
395
+
396
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
397
+ decorator will execute a no-op task after all retries have been exhausted,
398
+ ensuring that the flow execution can continue.
376
399
 
377
400
 
378
401
  Parameters
379
402
  ----------
380
- vars : Dict[str, str], default {}
381
- Dictionary of environment variables to set.
403
+ times : int, default 3
404
+ Number of times to retry this task.
405
+ minutes_between_retries : int, default 2
406
+ Number of minutes between retries.
382
407
  """
383
408
  ...
384
409
 
385
410
  @typing.overload
386
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
387
- """
388
- Decorator prototype for all step decorators. This function gets specialized
389
- and imported for all decorators types by _import_plugin_decorators().
390
- """
411
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
391
412
  ...
392
413
 
393
414
  @typing.overload
394
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
415
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
395
416
  ...
396
417
 
397
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
418
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
398
419
  """
399
- Decorator prototype for all step decorators. This function gets specialized
400
- and imported for all decorators types by _import_plugin_decorators().
420
+ Specifies the number of times the task corresponding
421
+ to a step needs to be retried.
422
+
423
+ This decorator is useful for handling transient errors, such as networking issues.
424
+ If your task contains operations that can't be retried safely, e.g. database updates,
425
+ it is advisable to annotate it with `@retry(times=0)`.
426
+
427
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
428
+ decorator will execute a no-op task after all retries have been exhausted,
429
+ ensuring that the flow execution can continue.
430
+
431
+
432
+ Parameters
433
+ ----------
434
+ times : int, default 3
435
+ Number of times to retry this task.
436
+ minutes_between_retries : int, default 2
437
+ Number of minutes between retries.
401
438
  """
402
439
  ...
403
440
 
404
441
  @typing.overload
405
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
442
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
406
443
  """
407
- Specifies the PyPI packages for the step.
444
+ Specifies the Conda environment for the step.
408
445
 
409
446
  Information in this decorator will augment any
410
- attributes set in the `@pyi_base` flow-level decorator. Hence,
411
- you can use `@pypi_base` to set packages required by all
412
- steps and use `@pypi` to specify step-specific overrides.
447
+ attributes set in the `@conda_base` flow-level decorator. Hence,
448
+ you can use `@conda_base` to set packages required by all
449
+ steps and use `@conda` to specify step-specific overrides.
413
450
 
414
451
 
415
452
  Parameters
416
453
  ----------
417
- packages : Dict[str, str], default: {}
454
+ packages : Dict[str, str], default {}
418
455
  Packages to use for this step. The key is the name of the package
419
456
  and the value is the version to use.
420
- python : str, optional, default: None
457
+ libraries : Dict[str, str], default {}
458
+ Supported for backward compatibility. When used with packages, packages will take precedence.
459
+ python : str, optional, default None
421
460
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
422
461
  that the version used will correspond to the version of the Python interpreter used to start the run.
462
+ disabled : bool, default False
463
+ If set to True, disables @conda.
423
464
  """
424
465
  ...
425
466
 
426
467
  @typing.overload
427
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
468
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
428
469
  ...
429
470
 
430
471
  @typing.overload
431
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
472
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
432
473
  ...
433
474
 
434
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
475
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
435
476
  """
436
- Specifies the PyPI packages for the step.
477
+ Specifies the Conda environment for the step.
437
478
 
438
479
  Information in this decorator will augment any
439
- attributes set in the `@pyi_base` flow-level decorator. Hence,
440
- you can use `@pypi_base` to set packages required by all
441
- steps and use `@pypi` to specify step-specific overrides.
480
+ attributes set in the `@conda_base` flow-level decorator. Hence,
481
+ you can use `@conda_base` to set packages required by all
482
+ steps and use `@conda` to specify step-specific overrides.
442
483
 
443
484
 
444
485
  Parameters
445
486
  ----------
446
- packages : Dict[str, str], default: {}
487
+ packages : Dict[str, str], default {}
447
488
  Packages to use for this step. The key is the name of the package
448
489
  and the value is the version to use.
449
- python : str, optional, default: None
490
+ libraries : Dict[str, str], default {}
491
+ Supported for backward compatibility. When used with packages, packages will take precedence.
492
+ python : str, optional, default None
450
493
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
451
494
  that the version used will correspond to the version of the Python interpreter used to start the run.
495
+ disabled : bool, default False
496
+ If set to True, disables @conda.
497
+ """
498
+ ...
499
+
500
+ @typing.overload
501
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
502
+ """
503
+ Specifies the resources needed when executing this step.
504
+
505
+ Use `@resources` to specify the resource requirements
506
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
507
+
508
+ You can choose the compute layer on the command line by executing e.g.
509
+ ```
510
+ python myflow.py run --with batch
511
+ ```
512
+ or
513
+ ```
514
+ python myflow.py run --with kubernetes
515
+ ```
516
+ which executes the flow on the desired system using the
517
+ requirements specified in `@resources`.
518
+
519
+
520
+ Parameters
521
+ ----------
522
+ cpu : int, default 1
523
+ Number of CPUs required for this step.
524
+ gpu : int, optional, default None
525
+ Number of GPUs required for this step.
526
+ disk : int, optional, default None
527
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
528
+ memory : int, default 4096
529
+ Memory size (in MB) required for this step.
530
+ shared_memory : int, optional, default None
531
+ The value for the size (in MiB) of the /dev/shm volume for this step.
532
+ This parameter maps to the `--shm-size` option in Docker.
533
+ """
534
+ ...
535
+
536
+ @typing.overload
537
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
538
+ ...
539
+
540
+ @typing.overload
541
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
542
+ ...
543
+
544
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
545
+ """
546
+ Specifies the resources needed when executing this step.
547
+
548
+ Use `@resources` to specify the resource requirements
549
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
550
+
551
+ You can choose the compute layer on the command line by executing e.g.
552
+ ```
553
+ python myflow.py run --with batch
554
+ ```
555
+ or
556
+ ```
557
+ python myflow.py run --with kubernetes
558
+ ```
559
+ which executes the flow on the desired system using the
560
+ requirements specified in `@resources`.
561
+
562
+
563
+ Parameters
564
+ ----------
565
+ cpu : int, default 1
566
+ Number of CPUs required for this step.
567
+ gpu : int, optional, default None
568
+ Number of GPUs required for this step.
569
+ disk : int, optional, default None
570
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
571
+ memory : int, default 4096
572
+ Memory size (in MB) required for this step.
573
+ shared_memory : int, optional, default None
574
+ The value for the size (in MiB) of the /dev/shm volume for this step.
575
+ This parameter maps to the `--shm-size` option in Docker.
452
576
  """
453
577
  ...
454
578
 
@@ -503,113 +627,9 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
503
627
  """
504
628
  ...
505
629
 
506
- @typing.overload
507
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
508
- """
509
- Creates a human-readable report, a Metaflow Card, after this step completes.
510
-
511
- Note that you may add multiple `@card` decorators in a step with different parameters.
512
-
513
-
514
- Parameters
515
- ----------
516
- type : str, default 'default'
517
- Card type.
518
- id : str, optional, default None
519
- If multiple cards are present, use this id to identify this card.
520
- options : Dict[str, Any], default {}
521
- Options passed to the card. The contents depend on the card type.
522
- timeout : int, default 45
523
- Interrupt reporting if it takes more than this many seconds.
524
- """
525
- ...
526
-
527
- @typing.overload
528
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
529
- ...
530
-
531
- @typing.overload
532
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
533
- ...
534
-
535
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
630
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
536
631
  """
537
- Creates a human-readable report, a Metaflow Card, after this step completes.
538
-
539
- Note that you may add multiple `@card` decorators in a step with different parameters.
540
-
541
-
542
- Parameters
543
- ----------
544
- type : str, default 'default'
545
- Card type.
546
- id : str, optional, default None
547
- If multiple cards are present, use this id to identify this card.
548
- options : Dict[str, Any], default {}
549
- Options passed to the card. The contents depend on the card type.
550
- timeout : int, default 45
551
- Interrupt reporting if it takes more than this many seconds.
552
- """
553
- ...
554
-
555
- @typing.overload
556
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
557
- """
558
- Specifies the number of times the task corresponding
559
- to a step needs to be retried.
560
-
561
- This decorator is useful for handling transient errors, such as networking issues.
562
- If your task contains operations that can't be retried safely, e.g. database updates,
563
- it is advisable to annotate it with `@retry(times=0)`.
564
-
565
- This can be used in conjunction with the `@catch` decorator. The `@catch`
566
- decorator will execute a no-op task after all retries have been exhausted,
567
- ensuring that the flow execution can continue.
568
-
569
-
570
- Parameters
571
- ----------
572
- times : int, default 3
573
- Number of times to retry this task.
574
- minutes_between_retries : int, default 2
575
- Number of minutes between retries.
576
- """
577
- ...
578
-
579
- @typing.overload
580
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
581
- ...
582
-
583
- @typing.overload
584
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
585
- ...
586
-
587
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
588
- """
589
- Specifies the number of times the task corresponding
590
- to a step needs to be retried.
591
-
592
- This decorator is useful for handling transient errors, such as networking issues.
593
- If your task contains operations that can't be retried safely, e.g. database updates,
594
- it is advisable to annotate it with `@retry(times=0)`.
595
-
596
- This can be used in conjunction with the `@catch` decorator. The `@catch`
597
- decorator will execute a no-op task after all retries have been exhausted,
598
- ensuring that the flow execution can continue.
599
-
600
-
601
- Parameters
602
- ----------
603
- times : int, default 3
604
- Number of times to retry this task.
605
- minutes_between_retries : int, default 2
606
- Number of minutes between retries.
607
- """
608
- ...
609
-
610
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
611
- """
612
- Specifies that this step should execute on Kubernetes.
632
+ Specifies that this step should execute on Kubernetes.
613
633
 
614
634
 
615
635
  Parameters
@@ -697,190 +717,205 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
697
717
  ...
698
718
 
699
719
  @typing.overload
700
- def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
720
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
701
721
  """
702
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
722
+ Specifies secrets to be retrieved and injected as environment variables prior to
723
+ the execution of a step.
703
724
 
704
725
 
705
726
  Parameters
706
727
  ----------
707
- cpu : int, default 1
708
- Number of CPUs required for this step. If `@resources` is
709
- also present, the maximum value from all decorators is used.
710
- gpu : int, default 0
711
- Number of GPUs required for this step. If `@resources` is
712
- also present, the maximum value from all decorators is used.
713
- memory : int, default 4096
714
- Memory size (in MB) required for this step. If
715
- `@resources` is also present, the maximum value from all decorators is
716
- used.
717
- image : str, optional, default None
718
- Docker image to use when launching on AWS Batch. If not specified, and
719
- METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
720
- not, a default Docker image mapping to the current version of Python is used.
721
- queue : str, default METAFLOW_BATCH_JOB_QUEUE
722
- AWS Batch Job Queue to submit the job to.
723
- iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
724
- AWS IAM role that AWS Batch container uses to access AWS cloud resources.
725
- execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
726
- AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
727
- (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
728
- shared_memory : int, optional, default None
729
- The value for the size (in MiB) of the /dev/shm volume for this step.
730
- This parameter maps to the `--shm-size` option in Docker.
731
- max_swap : int, optional, default None
732
- The total amount of swap memory (in MiB) a container can use for this
733
- step. This parameter is translated to the `--memory-swap` option in
734
- Docker where the value is the sum of the container memory plus the
735
- `max_swap` value.
736
- swappiness : int, optional, default None
737
- This allows you to tune memory swappiness behavior for this step.
738
- A swappiness value of 0 causes swapping not to happen unless absolutely
739
- necessary. A swappiness value of 100 causes pages to be swapped very
740
- aggressively. Accepted values are whole numbers between 0 and 100.
741
- use_tmpfs : bool, default False
742
- This enables an explicit tmpfs mount for this step. Note that tmpfs is
743
- not available on Fargate compute environments
744
- tmpfs_tempdir : bool, default True
745
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
746
- tmpfs_size : int, optional, default None
747
- The value for the size (in MiB) of the tmpfs mount for this step.
748
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
749
- memory allocated for this step.
750
- tmpfs_path : str, optional, default None
751
- Path to tmpfs mount for this step. Defaults to /metaflow_temp.
752
- inferentia : int, default 0
753
- Number of Inferentia chips required for this step.
754
- trainium : int, default None
755
- Alias for inferentia. Use only one of the two.
756
- efa : int, default 0
757
- Number of elastic fabric adapter network devices to attach to container
758
- ephemeral_storage : int, default None
759
- The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
760
- This is only relevant for Fargate compute environments
761
- log_driver: str, optional, default None
762
- The log driver to use for the Amazon ECS container.
763
- log_options: List[str], optional, default None
764
- List of strings containing options for the chosen log driver. The configurable values
765
- depend on the `log driver` chosen. Validation of these options is not supported yet.
766
- Example: [`awslogs-group:aws/batch/job`]
728
+ sources : List[Union[str, Dict[str, Any]]], default: []
729
+ List of secret specs, defining how the secrets are to be retrieved
730
+ role : str, optional, default: None
731
+ Role to use for fetching secrets
767
732
  """
768
733
  ...
769
734
 
770
735
  @typing.overload
771
- def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
736
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
772
737
  ...
773
738
 
774
739
  @typing.overload
775
- def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
740
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
776
741
  ...
777
742
 
778
- def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
743
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
779
744
  """
780
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
745
+ Specifies secrets to be retrieved and injected as environment variables prior to
746
+ the execution of a step.
781
747
 
782
748
 
783
749
  Parameters
784
750
  ----------
785
- cpu : int, default 1
786
- Number of CPUs required for this step. If `@resources` is
787
- also present, the maximum value from all decorators is used.
788
- gpu : int, default 0
789
- Number of GPUs required for this step. If `@resources` is
790
- also present, the maximum value from all decorators is used.
791
- memory : int, default 4096
792
- Memory size (in MB) required for this step. If
793
- `@resources` is also present, the maximum value from all decorators is
794
- used.
795
- image : str, optional, default None
796
- Docker image to use when launching on AWS Batch. If not specified, and
797
- METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
798
- not, a default Docker image mapping to the current version of Python is used.
799
- queue : str, default METAFLOW_BATCH_JOB_QUEUE
800
- AWS Batch Job Queue to submit the job to.
801
- iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
802
- AWS IAM role that AWS Batch container uses to access AWS cloud resources.
803
- execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
804
- AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
805
- (https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
806
- shared_memory : int, optional, default None
807
- The value for the size (in MiB) of the /dev/shm volume for this step.
808
- This parameter maps to the `--shm-size` option in Docker.
809
- max_swap : int, optional, default None
810
- The total amount of swap memory (in MiB) a container can use for this
811
- step. This parameter is translated to the `--memory-swap` option in
812
- Docker where the value is the sum of the container memory plus the
813
- `max_swap` value.
814
- swappiness : int, optional, default None
815
- This allows you to tune memory swappiness behavior for this step.
816
- A swappiness value of 0 causes swapping not to happen unless absolutely
817
- necessary. A swappiness value of 100 causes pages to be swapped very
818
- aggressively. Accepted values are whole numbers between 0 and 100.
819
- use_tmpfs : bool, default False
820
- This enables an explicit tmpfs mount for this step. Note that tmpfs is
821
- not available on Fargate compute environments
822
- tmpfs_tempdir : bool, default True
823
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
824
- tmpfs_size : int, optional, default None
825
- The value for the size (in MiB) of the tmpfs mount for this step.
826
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
827
- memory allocated for this step.
828
- tmpfs_path : str, optional, default None
829
- Path to tmpfs mount for this step. Defaults to /metaflow_temp.
830
- inferentia : int, default 0
831
- Number of Inferentia chips required for this step.
832
- trainium : int, default None
833
- Alias for inferentia. Use only one of the two.
834
- efa : int, default 0
835
- Number of elastic fabric adapter network devices to attach to container
836
- ephemeral_storage : int, default None
837
- The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
838
- This is only relevant for Fargate compute environments
839
- log_driver: str, optional, default None
840
- The log driver to use for the Amazon ECS container.
841
- log_options: List[str], optional, default None
842
- List of strings containing options for the chosen log driver. The configurable values
843
- depend on the `log driver` chosen. Validation of these options is not supported yet.
844
- Example: [`awslogs-group:aws/batch/job`]
751
+ sources : List[Union[str, Dict[str, Any]]], default: []
752
+ List of secret specs, defining how the secrets are to be retrieved
753
+ role : str, optional, default: None
754
+ Role to use for fetching secrets
845
755
  """
846
756
  ...
847
757
 
848
758
  @typing.overload
849
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
759
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
850
760
  """
851
- Specifies secrets to be retrieved and injected as environment variables prior to
852
- the execution of a step.
761
+ Decorator prototype for all step decorators. This function gets specialized
762
+ and imported for all decorators types by _import_plugin_decorators().
763
+ """
764
+ ...
765
+
766
+ @typing.overload
767
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
768
+ ...
769
+
770
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
771
+ """
772
+ Decorator prototype for all step decorators. This function gets specialized
773
+ and imported for all decorators types by _import_plugin_decorators().
774
+ """
775
+ ...
776
+
777
+ @typing.overload
778
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
779
+ """
780
+ Specifies a timeout for your step.
781
+
782
+ This decorator is useful if this step may hang indefinitely.
783
+
784
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
785
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
786
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
787
+
788
+ Note that all the values specified in parameters are added together so if you specify
789
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
853
790
 
854
791
 
855
792
  Parameters
856
793
  ----------
857
- sources : List[Union[str, Dict[str, Any]]], default: []
858
- List of secret specs, defining how the secrets are to be retrieved
859
- role : str, optional, default: None
860
- Role to use for fetching secrets
794
+ seconds : int, default 0
795
+ Number of seconds to wait prior to timing out.
796
+ minutes : int, default 0
797
+ Number of minutes to wait prior to timing out.
798
+ hours : int, default 0
799
+ Number of hours to wait prior to timing out.
861
800
  """
862
801
  ...
863
802
 
864
803
  @typing.overload
865
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
804
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
866
805
  ...
867
806
 
868
807
  @typing.overload
869
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
808
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
870
809
  ...
871
810
 
872
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
811
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
873
812
  """
874
- Specifies secrets to be retrieved and injected as environment variables prior to
875
- the execution of a step.
813
+ Specifies a timeout for your step.
814
+
815
+ This decorator is useful if this step may hang indefinitely.
816
+
817
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
818
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
819
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
820
+
821
+ Note that all the values specified in parameters are added together so if you specify
822
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
876
823
 
877
824
 
878
825
  Parameters
879
826
  ----------
880
- sources : List[Union[str, Dict[str, Any]]], default: []
881
- List of secret specs, defining how the secrets are to be retrieved
882
- role : str, optional, default: None
883
- Role to use for fetching secrets
827
+ seconds : int, default 0
828
+ Number of seconds to wait prior to timing out.
829
+ minutes : int, default 0
830
+ Number of minutes to wait prior to timing out.
831
+ hours : int, default 0
832
+ Number of hours to wait prior to timing out.
833
+ """
834
+ ...
835
+
836
+ @typing.overload
837
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
838
+ """
839
+ Specifies the PyPI packages for the step.
840
+
841
+ Information in this decorator will augment any
842
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
843
+ you can use `@pypi_base` to set packages required by all
844
+ steps and use `@pypi` to specify step-specific overrides.
845
+
846
+
847
+ Parameters
848
+ ----------
849
+ packages : Dict[str, str], default: {}
850
+ Packages to use for this step. The key is the name of the package
851
+ and the value is the version to use.
852
+ python : str, optional, default: None
853
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
854
+ that the version used will correspond to the version of the Python interpreter used to start the run.
855
+ """
856
+ ...
857
+
858
+ @typing.overload
859
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
860
+ ...
861
+
862
+ @typing.overload
863
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
864
+ ...
865
+
866
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
867
+ """
868
+ Specifies the PyPI packages for the step.
869
+
870
+ Information in this decorator will augment any
871
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
872
+ you can use `@pypi_base` to set packages required by all
873
+ steps and use `@pypi` to specify step-specific overrides.
874
+
875
+
876
+ Parameters
877
+ ----------
878
+ packages : Dict[str, str], default: {}
879
+ Packages to use for this step. The key is the name of the package
880
+ and the value is the version to use.
881
+ python : str, optional, default: None
882
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
883
+ that the version used will correspond to the version of the Python interpreter used to start the run.
884
+ """
885
+ ...
886
+
887
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
888
+ """
889
+ Specifies what flows belong to the same project.
890
+
891
+ A project-specific namespace is created for all flows that
892
+ use the same `@project(name)`.
893
+
894
+
895
+ Parameters
896
+ ----------
897
+ name : str
898
+ Project name. Make sure that the name is unique amongst all
899
+ projects that use the same production scheduler. The name may
900
+ contain only lowercase alphanumeric characters and underscores.
901
+
902
+ branch : Optional[str], default None
903
+ The branch to use. If not specified, the branch is set to
904
+ `user.<username>` unless `production` is set to `True`. This can
905
+ also be set on the command line using `--branch` as a top-level option.
906
+ It is an error to specify `branch` in the decorator and on the command line.
907
+
908
+ production : bool, default False
909
+ Whether or not the branch is the production branch. This can also be set on the
910
+ command line using `--production` as a top-level option. It is an error to specify
911
+ `production` in the decorator and on the command line.
912
+ The project branch name will be:
913
+ - if `branch` is specified:
914
+ - if `production` is True: `prod.<branch>`
915
+ - if `production` is False: `test.<branch>`
916
+ - if `branch` is not specified:
917
+ - if `production` is True: `prod`
918
+ - if `production` is False: `user.<username>`
884
919
  """
885
920
  ...
886
921
 
@@ -911,17 +946,161 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
911
946
  """
912
947
  Specifies the PyPI packages for all steps of the flow.
913
948
 
914
- Use `@pypi_base` to set common packages required by all
915
- steps and use `@pypi` to specify step-specific overrides.
949
+ Use `@pypi_base` to set common packages required by all
950
+ steps and use `@pypi` to specify step-specific overrides.
951
+
952
+ Parameters
953
+ ----------
954
+ packages : Dict[str, str], default: {}
955
+ Packages to use for this flow. The key is the name of the package
956
+ and the value is the version to use.
957
+ python : str, optional, default: None
958
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
959
+ that the version used will correspond to the version of the Python interpreter used to start the run.
960
+ """
961
+ ...
962
+
963
+ @typing.overload
964
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
965
+ """
966
+ Specifies the Conda environment for all steps of the flow.
967
+
968
+ Use `@conda_base` to set common libraries required by all
969
+ steps and use `@conda` to specify step-specific additions.
970
+
971
+
972
+ Parameters
973
+ ----------
974
+ packages : Dict[str, str], default {}
975
+ Packages to use for this flow. The key is the name of the package
976
+ and the value is the version to use.
977
+ libraries : Dict[str, str], default {}
978
+ Supported for backward compatibility. When used with packages, packages will take precedence.
979
+ python : str, optional, default None
980
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
981
+ that the version used will correspond to the version of the Python interpreter used to start the run.
982
+ disabled : bool, default False
983
+ If set to True, disables Conda.
984
+ """
985
+ ...
986
+
987
+ @typing.overload
988
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
989
+ ...
990
+
991
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
992
+ """
993
+ Specifies the Conda environment for all steps of the flow.
994
+
995
+ Use `@conda_base` to set common libraries required by all
996
+ steps and use `@conda` to specify step-specific additions.
997
+
998
+
999
+ Parameters
1000
+ ----------
1001
+ packages : Dict[str, str], default {}
1002
+ Packages to use for this flow. The key is the name of the package
1003
+ and the value is the version to use.
1004
+ libraries : Dict[str, str], default {}
1005
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1006
+ python : str, optional, default None
1007
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1008
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1009
+ disabled : bool, default False
1010
+ If set to True, disables Conda.
1011
+ """
1012
+ ...
1013
+
1014
+ @typing.overload
1015
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1016
+ """
1017
+ Specifies the event(s) that this flow depends on.
1018
+
1019
+ ```
1020
+ @trigger(event='foo')
1021
+ ```
1022
+ or
1023
+ ```
1024
+ @trigger(events=['foo', 'bar'])
1025
+ ```
1026
+
1027
+ Additionally, you can specify the parameter mappings
1028
+ to map event payload to Metaflow parameters for the flow.
1029
+ ```
1030
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1031
+ ```
1032
+ or
1033
+ ```
1034
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1035
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1036
+ ```
1037
+
1038
+ 'parameters' can also be a list of strings and tuples like so:
1039
+ ```
1040
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1041
+ ```
1042
+ This is equivalent to:
1043
+ ```
1044
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1045
+ ```
1046
+
1047
+
1048
+ Parameters
1049
+ ----------
1050
+ event : Union[str, Dict[str, Any]], optional, default None
1051
+ Event dependency for this flow.
1052
+ events : List[Union[str, Dict[str, Any]]], default []
1053
+ Events dependency for this flow.
1054
+ options : Dict[str, Any], default {}
1055
+ Backend-specific configuration for tuning eventing behavior.
1056
+ """
1057
+ ...
1058
+
1059
+ @typing.overload
1060
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1061
+ ...
1062
+
1063
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1064
+ """
1065
+ Specifies the event(s) that this flow depends on.
1066
+
1067
+ ```
1068
+ @trigger(event='foo')
1069
+ ```
1070
+ or
1071
+ ```
1072
+ @trigger(events=['foo', 'bar'])
1073
+ ```
1074
+
1075
+ Additionally, you can specify the parameter mappings
1076
+ to map event payload to Metaflow parameters for the flow.
1077
+ ```
1078
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1079
+ ```
1080
+ or
1081
+ ```
1082
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1083
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1084
+ ```
1085
+
1086
+ 'parameters' can also be a list of strings and tuples like so:
1087
+ ```
1088
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1089
+ ```
1090
+ This is equivalent to:
1091
+ ```
1092
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1093
+ ```
1094
+
916
1095
 
917
1096
  Parameters
918
1097
  ----------
919
- packages : Dict[str, str], default: {}
920
- Packages to use for this flow. The key is the name of the package
921
- and the value is the version to use.
922
- python : str, optional, default: None
923
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
924
- that the version used will correspond to the version of the Python interpreter used to start the run.
1098
+ event : Union[str, Dict[str, Any]], optional, default None
1099
+ Event dependency for this flow.
1100
+ events : List[Union[str, Dict[str, Any]]], default []
1101
+ Events dependency for this flow.
1102
+ options : Dict[str, Any], default {}
1103
+ Backend-specific configuration for tuning eventing behavior.
925
1104
  """
926
1105
  ...
927
1106
 
@@ -968,92 +1147,6 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
968
1147
  """
969
1148
  ...
970
1149
 
971
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
972
- """
973
- Specifies what flows belong to the same project.
974
-
975
- A project-specific namespace is created for all flows that
976
- use the same `@project(name)`.
977
-
978
-
979
- Parameters
980
- ----------
981
- name : str
982
- Project name. Make sure that the name is unique amongst all
983
- projects that use the same production scheduler. The name may
984
- contain only lowercase alphanumeric characters and underscores.
985
-
986
- branch : Optional[str], default None
987
- The branch to use. If not specified, the branch is set to
988
- `user.<username>` unless `production` is set to `True`. This can
989
- also be set on the command line using `--branch` as a top-level option.
990
- It is an error to specify `branch` in the decorator and on the command line.
991
-
992
- production : bool, default False
993
- Whether or not the branch is the production branch. This can also be set on the
994
- command line using `--production` as a top-level option. It is an error to specify
995
- `production` in the decorator and on the command line.
996
- The project branch name will be:
997
- - if `branch` is specified:
998
- - if `production` is True: `prod.<branch>`
999
- - if `production` is False: `test.<branch>`
1000
- - if `branch` is not specified:
1001
- - if `production` is True: `prod`
1002
- - if `production` is False: `user.<username>`
1003
- """
1004
- ...
1005
-
1006
- @typing.overload
1007
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1008
- """
1009
- Specifies the times when the flow should be run when running on a
1010
- production scheduler.
1011
-
1012
-
1013
- Parameters
1014
- ----------
1015
- hourly : bool, default False
1016
- Run the workflow hourly.
1017
- daily : bool, default True
1018
- Run the workflow daily.
1019
- weekly : bool, default False
1020
- Run the workflow weekly.
1021
- cron : str, optional, default None
1022
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1023
- specified by this expression.
1024
- timezone : str, optional, default None
1025
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1026
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1027
- """
1028
- ...
1029
-
1030
- @typing.overload
1031
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1032
- ...
1033
-
1034
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1035
- """
1036
- Specifies the times when the flow should be run when running on a
1037
- production scheduler.
1038
-
1039
-
1040
- Parameters
1041
- ----------
1042
- hourly : bool, default False
1043
- Run the workflow hourly.
1044
- daily : bool, default True
1045
- Run the workflow daily.
1046
- weekly : bool, default False
1047
- Run the workflow weekly.
1048
- cron : str, optional, default None
1049
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1050
- specified by this expression.
1051
- timezone : str, optional, default None
1052
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1053
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1054
- """
1055
- ...
1056
-
1057
1150
  def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1058
1151
  """
1059
1152
  The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
@@ -1098,95 +1191,53 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1098
1191
  ...
1099
1192
 
1100
1193
  @typing.overload
1101
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1194
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1102
1195
  """
1103
- Specifies the event(s) that this flow depends on.
1104
-
1105
- ```
1106
- @trigger(event='foo')
1107
- ```
1108
- or
1109
- ```
1110
- @trigger(events=['foo', 'bar'])
1111
- ```
1112
-
1113
- Additionally, you can specify the parameter mappings
1114
- to map event payload to Metaflow parameters for the flow.
1115
- ```
1116
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1117
- ```
1118
- or
1119
- ```
1120
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1121
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1122
- ```
1123
-
1124
- 'parameters' can also be a list of strings and tuples like so:
1125
- ```
1126
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1127
- ```
1128
- This is equivalent to:
1129
- ```
1130
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1131
- ```
1196
+ Specifies the times when the flow should be run when running on a
1197
+ production scheduler.
1132
1198
 
1133
1199
 
1134
1200
  Parameters
1135
1201
  ----------
1136
- event : Union[str, Dict[str, Any]], optional, default None
1137
- Event dependency for this flow.
1138
- events : List[Union[str, Dict[str, Any]]], default []
1139
- Events dependency for this flow.
1140
- options : Dict[str, Any], default {}
1141
- Backend-specific configuration for tuning eventing behavior.
1202
+ hourly : bool, default False
1203
+ Run the workflow hourly.
1204
+ daily : bool, default True
1205
+ Run the workflow daily.
1206
+ weekly : bool, default False
1207
+ Run the workflow weekly.
1208
+ cron : str, optional, default None
1209
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1210
+ specified by this expression.
1211
+ timezone : str, optional, default None
1212
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1213
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1142
1214
  """
1143
1215
  ...
1144
1216
 
1145
1217
  @typing.overload
1146
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1218
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1147
1219
  ...
1148
1220
 
1149
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1221
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1150
1222
  """
1151
- Specifies the event(s) that this flow depends on.
1152
-
1153
- ```
1154
- @trigger(event='foo')
1155
- ```
1156
- or
1157
- ```
1158
- @trigger(events=['foo', 'bar'])
1159
- ```
1160
-
1161
- Additionally, you can specify the parameter mappings
1162
- to map event payload to Metaflow parameters for the flow.
1163
- ```
1164
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1165
- ```
1166
- or
1167
- ```
1168
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1169
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1170
- ```
1171
-
1172
- 'parameters' can also be a list of strings and tuples like so:
1173
- ```
1174
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1175
- ```
1176
- This is equivalent to:
1177
- ```
1178
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1179
- ```
1223
+ Specifies the times when the flow should be run when running on a
1224
+ production scheduler.
1180
1225
 
1181
1226
 
1182
1227
  Parameters
1183
1228
  ----------
1184
- event : Union[str, Dict[str, Any]], optional, default None
1185
- Event dependency for this flow.
1186
- events : List[Union[str, Dict[str, Any]]], default []
1187
- Events dependency for this flow.
1188
- options : Dict[str, Any], default {}
1189
- Backend-specific configuration for tuning eventing behavior.
1229
+ hourly : bool, default False
1230
+ Run the workflow hourly.
1231
+ daily : bool, default True
1232
+ Run the workflow daily.
1233
+ weekly : bool, default False
1234
+ Run the workflow weekly.
1235
+ cron : str, optional, default None
1236
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1237
+ specified by this expression.
1238
+ timezone : str, optional, default None
1239
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1240
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1190
1241
  """
1191
1242
  ...
1192
1243
 
@@ -1291,54 +1342,3 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1291
1342
  """
1292
1343
  ...
1293
1344
 
1294
- @typing.overload
1295
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1296
- """
1297
- Specifies the Conda environment for all steps of the flow.
1298
-
1299
- Use `@conda_base` to set common libraries required by all
1300
- steps and use `@conda` to specify step-specific additions.
1301
-
1302
-
1303
- Parameters
1304
- ----------
1305
- packages : Dict[str, str], default {}
1306
- Packages to use for this flow. The key is the name of the package
1307
- and the value is the version to use.
1308
- libraries : Dict[str, str], default {}
1309
- Supported for backward compatibility. When used with packages, packages will take precedence.
1310
- python : str, optional, default None
1311
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1312
- that the version used will correspond to the version of the Python interpreter used to start the run.
1313
- disabled : bool, default False
1314
- If set to True, disables Conda.
1315
- """
1316
- ...
1317
-
1318
- @typing.overload
1319
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1320
- ...
1321
-
1322
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1323
- """
1324
- Specifies the Conda environment for all steps of the flow.
1325
-
1326
- Use `@conda_base` to set common libraries required by all
1327
- steps and use `@conda` to specify step-specific additions.
1328
-
1329
-
1330
- Parameters
1331
- ----------
1332
- packages : Dict[str, str], default {}
1333
- Packages to use for this flow. The key is the name of the package
1334
- and the value is the version to use.
1335
- libraries : Dict[str, str], default {}
1336
- Supported for backward compatibility. When used with packages, packages will take precedence.
1337
- python : str, optional, default None
1338
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1339
- that the version used will correspond to the version of the Python interpreter used to start the run.
1340
- disabled : bool, default False
1341
- If set to True, disables Conda.
1342
- """
1343
- ...
1344
-