metaflow-stubs 2.18.0__py2.py3-none-any.whl → 2.18.2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of metaflow-stubs might be problematic. Click here for more details.

Files changed (166) hide show
  1. metaflow-stubs/__init__.pyi +598 -592
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +6 -6
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +4 -2
  20. metaflow-stubs/metaflow_current.pyi +28 -28
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/multicore_utils.pyi +2 -2
  23. metaflow-stubs/packaging_sys/__init__.pyi +6 -6
  24. metaflow-stubs/packaging_sys/backend.pyi +3 -3
  25. metaflow-stubs/packaging_sys/distribution_support.pyi +3 -3
  26. metaflow-stubs/packaging_sys/tar_backend.pyi +6 -6
  27. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  28. metaflow-stubs/packaging_sys/v1.pyi +4 -4
  29. metaflow-stubs/parameters.pyi +4 -4
  30. metaflow-stubs/plugins/__init__.pyi +14 -14
  31. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  33. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  34. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  35. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  37. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  38. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  39. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  41. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  42. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  43. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  44. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  45. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -4
  46. metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
  47. metaflow-stubs/plugins/aws/__init__.pyi +4 -4
  48. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/aws_utils.pyi +5 -2
  50. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  51. metaflow-stubs/plugins/aws/batch/batch.pyi +4 -4
  52. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  53. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +8 -2
  54. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  55. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  56. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  58. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  59. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  60. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +3 -3
  61. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  62. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  63. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  64. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  65. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  67. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  68. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  69. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  70. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  71. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  72. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  77. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  81. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  84. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  85. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  86. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  87. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  88. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  92. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  93. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  94. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  95. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  96. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  104. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  105. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  106. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  107. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  108. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  110. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  112. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  114. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  115. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  116. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  119. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  121. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  122. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  123. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  124. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  125. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  126. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  127. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  128. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  129. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  130. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  131. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  132. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  133. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  134. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  135. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  136. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  138. metaflow-stubs/pylint_wrapper.pyi +2 -2
  139. metaflow-stubs/runner/__init__.pyi +2 -2
  140. metaflow-stubs/runner/deployer.pyi +33 -33
  141. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  142. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  143. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  144. metaflow-stubs/runner/nbrun.pyi +2 -2
  145. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  146. metaflow-stubs/runner/utils.pyi +4 -4
  147. metaflow-stubs/system/__init__.pyi +2 -2
  148. metaflow-stubs/system/system_logger.pyi +3 -3
  149. metaflow-stubs/system/system_monitor.pyi +2 -2
  150. metaflow-stubs/tagging_util.pyi +2 -2
  151. metaflow-stubs/tuple_util.pyi +2 -2
  152. metaflow-stubs/user_configs/__init__.pyi +2 -2
  153. metaflow-stubs/user_configs/config_options.pyi +4 -4
  154. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  155. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  156. metaflow-stubs/user_decorators/common.pyi +2 -2
  157. metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
  158. metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
  159. metaflow-stubs/user_decorators/user_flow_decorator.pyi +4 -4
  160. metaflow-stubs/user_decorators/user_step_decorator.pyi +6 -6
  161. metaflow-stubs/version.pyi +2 -2
  162. {metaflow_stubs-2.18.0.dist-info → metaflow_stubs-2.18.2.dist-info}/METADATA +2 -2
  163. metaflow_stubs-2.18.2.dist-info/RECORD +166 -0
  164. metaflow_stubs-2.18.0.dist-info/RECORD +0 -166
  165. {metaflow_stubs-2.18.0.dist-info → metaflow_stubs-2.18.2.dist-info}/WHEEL +0 -0
  166. {metaflow_stubs-2.18.0.dist-info → metaflow_stubs-2.18.2.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.18.0 #
4
- # Generated on 2025-08-27T01:57:08.613943 #
3
+ # MF version: 2.18.2 #
4
+ # Generated on 2025-09-03T10:37:45.851135 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -40,15 +40,15 @@ from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
42
  from . import events as events
43
- from . import metaflow_git as metaflow_git
44
43
  from . import tuple_util as tuple_util
44
+ from . import metaflow_git as metaflow_git
45
45
  from . import runner as runner
46
46
  from . import plugins as plugins
47
47
  from .plugins.datatools.s3.s3 import S3 as S3
48
48
  from . import includefile as includefile
49
49
  from .includefile import IncludeFile as IncludeFile
50
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
51
50
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
51
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
52
52
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
53
53
  from . import cards as cards
54
54
  from . import client as client
@@ -152,217 +152,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
152
152
  """
153
153
  ...
154
154
 
155
- @typing.overload
156
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
157
- """
158
- Decorator prototype for all step decorators. This function gets specialized
159
- and imported for all decorators types by _import_plugin_decorators().
160
- """
161
- ...
162
-
163
- @typing.overload
164
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
165
- ...
166
-
167
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
168
- """
169
- Decorator prototype for all step decorators. This function gets specialized
170
- and imported for all decorators types by _import_plugin_decorators().
171
- """
172
- ...
173
-
174
- @typing.overload
175
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
176
- """
177
- Creates a human-readable report, a Metaflow Card, after this step completes.
178
-
179
- Note that you may add multiple `@card` decorators in a step with different parameters.
180
-
181
-
182
- Parameters
183
- ----------
184
- type : str, default 'default'
185
- Card type.
186
- id : str, optional, default None
187
- If multiple cards are present, use this id to identify this card.
188
- options : Dict[str, Any], default {}
189
- Options passed to the card. The contents depend on the card type.
190
- timeout : int, default 45
191
- Interrupt reporting if it takes more than this many seconds.
192
- """
193
- ...
194
-
195
- @typing.overload
196
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
197
- ...
198
-
199
- @typing.overload
200
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
201
- ...
202
-
203
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
204
- """
205
- Creates a human-readable report, a Metaflow Card, after this step completes.
206
-
207
- Note that you may add multiple `@card` decorators in a step with different parameters.
208
-
209
-
210
- Parameters
211
- ----------
212
- type : str, default 'default'
213
- Card type.
214
- id : str, optional, default None
215
- If multiple cards are present, use this id to identify this card.
216
- options : Dict[str, Any], default {}
217
- Options passed to the card. The contents depend on the card type.
218
- timeout : int, default 45
219
- Interrupt reporting if it takes more than this many seconds.
220
- """
221
- ...
222
-
223
- @typing.overload
224
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
225
- """
226
- Specifies the PyPI packages for the step.
227
-
228
- Information in this decorator will augment any
229
- attributes set in the `@pyi_base` flow-level decorator. Hence,
230
- you can use `@pypi_base` to set packages required by all
231
- steps and use `@pypi` to specify step-specific overrides.
232
-
233
-
234
- Parameters
235
- ----------
236
- packages : Dict[str, str], default: {}
237
- Packages to use for this step. The key is the name of the package
238
- and the value is the version to use.
239
- python : str, optional, default: None
240
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
241
- that the version used will correspond to the version of the Python interpreter used to start the run.
242
- """
243
- ...
244
-
245
- @typing.overload
246
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
247
- ...
248
-
249
- @typing.overload
250
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
251
- ...
252
-
253
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
254
- """
255
- Specifies the PyPI packages for the step.
256
-
257
- Information in this decorator will augment any
258
- attributes set in the `@pyi_base` flow-level decorator. Hence,
259
- you can use `@pypi_base` to set packages required by all
260
- steps and use `@pypi` to specify step-specific overrides.
261
-
262
-
263
- Parameters
264
- ----------
265
- packages : Dict[str, str], default: {}
266
- Packages to use for this step. The key is the name of the package
267
- and the value is the version to use.
268
- python : str, optional, default: None
269
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
270
- that the version used will correspond to the version of the Python interpreter used to start the run.
271
- """
272
- ...
273
-
274
- @typing.overload
275
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
276
- """
277
- Specifies a timeout for your step.
278
-
279
- This decorator is useful if this step may hang indefinitely.
280
-
281
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
282
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
283
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
284
-
285
- Note that all the values specified in parameters are added together so if you specify
286
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
287
-
288
-
289
- Parameters
290
- ----------
291
- seconds : int, default 0
292
- Number of seconds to wait prior to timing out.
293
- minutes : int, default 0
294
- Number of minutes to wait prior to timing out.
295
- hours : int, default 0
296
- Number of hours to wait prior to timing out.
297
- """
298
- ...
299
-
300
- @typing.overload
301
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
302
- ...
303
-
304
- @typing.overload
305
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
306
- ...
307
-
308
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
309
- """
310
- Specifies a timeout for your step.
311
-
312
- This decorator is useful if this step may hang indefinitely.
313
-
314
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
315
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
316
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
317
-
318
- Note that all the values specified in parameters are added together so if you specify
319
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
320
-
321
-
322
- Parameters
323
- ----------
324
- seconds : int, default 0
325
- Number of seconds to wait prior to timing out.
326
- minutes : int, default 0
327
- Number of minutes to wait prior to timing out.
328
- hours : int, default 0
329
- Number of hours to wait prior to timing out.
330
- """
331
- ...
332
-
333
- @typing.overload
334
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
335
- """
336
- Specifies environment variables to be set prior to the execution of a step.
337
-
338
-
339
- Parameters
340
- ----------
341
- vars : Dict[str, str], default {}
342
- Dictionary of environment variables to set.
343
- """
344
- ...
345
-
346
- @typing.overload
347
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
348
- ...
349
-
350
- @typing.overload
351
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
352
- ...
353
-
354
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
355
- """
356
- Specifies environment variables to be set prior to the execution of a step.
357
-
358
-
359
- Parameters
360
- ----------
361
- vars : Dict[str, str], default {}
362
- Dictionary of environment variables to set.
363
- """
364
- ...
365
-
366
155
  @typing.overload
367
156
  def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
368
157
  """
@@ -443,189 +232,41 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
443
232
  ...
444
233
 
445
234
  @typing.overload
446
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
235
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
447
236
  """
448
- Specifies the Conda environment for the step.
237
+ Specifies the number of times the task corresponding
238
+ to a step needs to be retried.
449
239
 
450
- Information in this decorator will augment any
451
- attributes set in the `@conda_base` flow-level decorator. Hence,
452
- you can use `@conda_base` to set packages required by all
453
- steps and use `@conda` to specify step-specific overrides.
240
+ This decorator is useful for handling transient errors, such as networking issues.
241
+ If your task contains operations that can't be retried safely, e.g. database updates,
242
+ it is advisable to annotate it with `@retry(times=0)`.
243
+
244
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
245
+ decorator will execute a no-op task after all retries have been exhausted,
246
+ ensuring that the flow execution can continue.
454
247
 
455
248
 
456
249
  Parameters
457
250
  ----------
458
- packages : Dict[str, str], default {}
459
- Packages to use for this step. The key is the name of the package
460
- and the value is the version to use.
461
- libraries : Dict[str, str], default {}
462
- Supported for backward compatibility. When used with packages, packages will take precedence.
463
- python : str, optional, default None
464
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
465
- that the version used will correspond to the version of the Python interpreter used to start the run.
466
- disabled : bool, default False
467
- If set to True, disables @conda.
251
+ times : int, default 3
252
+ Number of times to retry this task.
253
+ minutes_between_retries : int, default 2
254
+ Number of minutes between retries.
468
255
  """
469
256
  ...
470
257
 
471
258
  @typing.overload
472
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
259
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
473
260
  ...
474
261
 
475
262
  @typing.overload
476
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
263
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
477
264
  ...
478
265
 
479
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
266
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
480
267
  """
481
- Specifies the Conda environment for the step.
482
-
483
- Information in this decorator will augment any
484
- attributes set in the `@conda_base` flow-level decorator. Hence,
485
- you can use `@conda_base` to set packages required by all
486
- steps and use `@conda` to specify step-specific overrides.
487
-
488
-
489
- Parameters
490
- ----------
491
- packages : Dict[str, str], default {}
492
- Packages to use for this step. The key is the name of the package
493
- and the value is the version to use.
494
- libraries : Dict[str, str], default {}
495
- Supported for backward compatibility. When used with packages, packages will take precedence.
496
- python : str, optional, default None
497
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
498
- that the version used will correspond to the version of the Python interpreter used to start the run.
499
- disabled : bool, default False
500
- If set to True, disables @conda.
501
- """
502
- ...
503
-
504
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
505
- """
506
- Specifies that this step should execute on Kubernetes.
507
-
508
-
509
- Parameters
510
- ----------
511
- cpu : int, default 1
512
- Number of CPUs required for this step. If `@resources` is
513
- also present, the maximum value from all decorators is used.
514
- memory : int, default 4096
515
- Memory size (in MB) required for this step. If
516
- `@resources` is also present, the maximum value from all decorators is
517
- used.
518
- disk : int, default 10240
519
- Disk size (in MB) required for this step. If
520
- `@resources` is also present, the maximum value from all decorators is
521
- used.
522
- image : str, optional, default None
523
- Docker image to use when launching on Kubernetes. If not specified, and
524
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
525
- not, a default Docker image mapping to the current version of Python is used.
526
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
527
- If given, the imagePullPolicy to be applied to the Docker image of the step.
528
- image_pull_secrets: List[str], default []
529
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
530
- Kubernetes image pull secrets to use when pulling container images
531
- in Kubernetes.
532
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
533
- Kubernetes service account to use when launching pod in Kubernetes.
534
- secrets : List[str], optional, default None
535
- Kubernetes secrets to use when launching pod in Kubernetes. These
536
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
537
- in Metaflow configuration.
538
- node_selector: Union[Dict[str,str], str], optional, default None
539
- Kubernetes node selector(s) to apply to the pod running the task.
540
- Can be passed in as a comma separated string of values e.g.
541
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
542
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
543
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
544
- Kubernetes namespace to use when launching pod in Kubernetes.
545
- gpu : int, optional, default None
546
- Number of GPUs required for this step. A value of zero implies that
547
- the scheduled node should not have GPUs.
548
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
549
- The vendor of the GPUs to be used for this step.
550
- tolerations : List[Dict[str,str]], default []
551
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
552
- Kubernetes tolerations to use when launching pod in Kubernetes.
553
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
554
- Kubernetes labels to use when launching pod in Kubernetes.
555
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
556
- Kubernetes annotations to use when launching pod in Kubernetes.
557
- use_tmpfs : bool, default False
558
- This enables an explicit tmpfs mount for this step.
559
- tmpfs_tempdir : bool, default True
560
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
561
- tmpfs_size : int, optional, default: None
562
- The value for the size (in MiB) of the tmpfs mount for this step.
563
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
564
- memory allocated for this step.
565
- tmpfs_path : str, optional, default /metaflow_temp
566
- Path to tmpfs mount for this step.
567
- persistent_volume_claims : Dict[str, str], optional, default None
568
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
569
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
570
- shared_memory: int, optional
571
- Shared memory size (in MiB) required for this step
572
- port: int, optional
573
- Port number to specify in the Kubernetes job object
574
- compute_pool : str, optional, default None
575
- Compute pool to be used for for this step.
576
- If not specified, any accessible compute pool within the perimeter is used.
577
- hostname_resolution_timeout: int, default 10 * 60
578
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
579
- Only applicable when @parallel is used.
580
- qos: str, default: Burstable
581
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
582
-
583
- security_context: Dict[str, Any], optional, default None
584
- Container security context. Applies to the task container. Allows the following keys:
585
- - privileged: bool, optional, default None
586
- - allow_privilege_escalation: bool, optional, default None
587
- - run_as_user: int, optional, default None
588
- - run_as_group: int, optional, default None
589
- - run_as_non_root: bool, optional, default None
590
- """
591
- ...
592
-
593
- @typing.overload
594
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
595
- """
596
- Specifies the number of times the task corresponding
597
- to a step needs to be retried.
598
-
599
- This decorator is useful for handling transient errors, such as networking issues.
600
- If your task contains operations that can't be retried safely, e.g. database updates,
601
- it is advisable to annotate it with `@retry(times=0)`.
602
-
603
- This can be used in conjunction with the `@catch` decorator. The `@catch`
604
- decorator will execute a no-op task after all retries have been exhausted,
605
- ensuring that the flow execution can continue.
606
-
607
-
608
- Parameters
609
- ----------
610
- times : int, default 3
611
- Number of times to retry this task.
612
- minutes_between_retries : int, default 2
613
- Number of minutes between retries.
614
- """
615
- ...
616
-
617
- @typing.overload
618
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
619
- ...
620
-
621
- @typing.overload
622
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
623
- ...
624
-
625
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
626
- """
627
- Specifies the number of times the task corresponding
628
- to a step needs to be retried.
268
+ Specifies the number of times the task corresponding
269
+ to a step needs to be retried.
629
270
 
630
271
  This decorator is useful for handling transient errors, such as networking issues.
631
272
  If your task contains operations that can't be retried safely, e.g. database updates,
@@ -646,7 +287,7 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
646
287
  ...
647
288
 
648
289
  @typing.overload
649
- def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
290
+ def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, aws_batch_tags: typing.Optional[typing.Dict[str, str]] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
650
291
  """
651
292
  Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
652
293
 
@@ -687,6 +328,9 @@ def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optio
687
328
  A swappiness value of 0 causes swapping not to happen unless absolutely
688
329
  necessary. A swappiness value of 100 causes pages to be swapped very
689
330
  aggressively. Accepted values are whole numbers between 0 and 100.
331
+ aws_batch_tags: Dict[str, str], optional, default None
332
+ Sets arbitrary AWS tags on the AWS Batch compute environment.
333
+ Set as string key-value pairs.
690
334
  use_tmpfs : bool, default False
691
335
  This enables an explicit tmpfs mount for this step. Note that tmpfs is
692
336
  not available on Fargate compute environments
@@ -724,7 +368,7 @@ def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Calla
724
368
  def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
725
369
  ...
726
370
 
727
- def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
371
+ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, aws_batch_tags: typing.Optional[typing.Dict[str, str]] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
728
372
  """
729
373
  Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
730
374
 
@@ -765,6 +409,9 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
765
409
  A swappiness value of 0 causes swapping not to happen unless absolutely
766
410
  necessary. A swappiness value of 100 causes pages to be swapped very
767
411
  aggressively. Accepted values are whole numbers between 0 and 100.
412
+ aws_batch_tags: Dict[str, str], optional, default None
413
+ Sets arbitrary AWS tags on the AWS Batch compute environment.
414
+ Set as string key-value pairs.
768
415
  use_tmpfs : bool, default False
769
416
  This enables an explicit tmpfs mount for this step. Note that tmpfs is
770
417
  not available on Fargate compute environments
@@ -795,107 +442,98 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
795
442
  ...
796
443
 
797
444
  @typing.overload
798
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
445
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
799
446
  """
800
- Specifies secrets to be retrieved and injected as environment variables prior to
801
- the execution of a step.
447
+ Creates a human-readable report, a Metaflow Card, after this step completes.
448
+
449
+ Note that you may add multiple `@card` decorators in a step with different parameters.
802
450
 
803
451
 
804
452
  Parameters
805
453
  ----------
806
- sources : List[Union[str, Dict[str, Any]]], default: []
807
- List of secret specs, defining how the secrets are to be retrieved
808
- role : str, optional, default: None
809
- Role to use for fetching secrets
454
+ type : str, default 'default'
455
+ Card type.
456
+ id : str, optional, default None
457
+ If multiple cards are present, use this id to identify this card.
458
+ options : Dict[str, Any], default {}
459
+ Options passed to the card. The contents depend on the card type.
460
+ timeout : int, default 45
461
+ Interrupt reporting if it takes more than this many seconds.
810
462
  """
811
463
  ...
812
464
 
813
465
  @typing.overload
814
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
466
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
815
467
  ...
816
468
 
817
469
  @typing.overload
818
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
470
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
819
471
  ...
820
472
 
821
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
473
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
822
474
  """
823
- Specifies secrets to be retrieved and injected as environment variables prior to
824
- the execution of a step.
475
+ Creates a human-readable report, a Metaflow Card, after this step completes.
476
+
477
+ Note that you may add multiple `@card` decorators in a step with different parameters.
825
478
 
826
479
 
827
480
  Parameters
828
481
  ----------
829
- sources : List[Union[str, Dict[str, Any]]], default: []
830
- List of secret specs, defining how the secrets are to be retrieved
831
- role : str, optional, default: None
832
- Role to use for fetching secrets
482
+ type : str, default 'default'
483
+ Card type.
484
+ id : str, optional, default None
485
+ If multiple cards are present, use this id to identify this card.
486
+ options : Dict[str, Any], default {}
487
+ Options passed to the card. The contents depend on the card type.
488
+ timeout : int, default 45
489
+ Interrupt reporting if it takes more than this many seconds.
833
490
  """
834
491
  ...
835
492
 
836
493
  @typing.overload
837
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
494
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
838
495
  """
839
- Specifies that the step will success under all circumstances.
496
+ Specifies the PyPI packages for the step.
840
497
 
841
- The decorator will create an optional artifact, specified by `var`, which
842
- contains the exception raised. You can use it to detect the presence
843
- of errors, indicating that all happy-path artifacts produced by the step
844
- are missing.
498
+ Information in this decorator will augment any
499
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
500
+ you can use `@pypi_base` to set packages required by all
501
+ steps and use `@pypi` to specify step-specific overrides.
845
502
 
846
503
 
847
504
  Parameters
848
505
  ----------
849
- var : str, optional, default None
850
- Name of the artifact in which to store the caught exception.
851
- If not specified, the exception is not stored.
852
- print_exception : bool, default True
853
- Determines whether or not the exception is printed to
854
- stdout when caught.
506
+ packages : Dict[str, str], default: {}
507
+ Packages to use for this step. The key is the name of the package
508
+ and the value is the version to use.
509
+ python : str, optional, default: None
510
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
511
+ that the version used will correspond to the version of the Python interpreter used to start the run.
855
512
  """
856
513
  ...
857
514
 
858
515
  @typing.overload
859
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
516
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
860
517
  ...
861
518
 
862
519
  @typing.overload
863
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
864
- ...
865
-
866
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
867
- """
868
- Specifies that the step will success under all circumstances.
869
-
870
- The decorator will create an optional artifact, specified by `var`, which
871
- contains the exception raised. You can use it to detect the presence
872
- of errors, indicating that all happy-path artifacts produced by the step
873
- are missing.
874
-
875
-
876
- Parameters
877
- ----------
878
- var : str, optional, default None
879
- Name of the artifact in which to store the caught exception.
880
- If not specified, the exception is not stored.
881
- print_exception : bool, default True
882
- Determines whether or not the exception is printed to
883
- stdout when caught.
884
- """
520
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
885
521
  ...
886
522
 
887
- @typing.overload
888
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
523
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
889
524
  """
890
- Specifies the PyPI packages for all steps of the flow.
525
+ Specifies the PyPI packages for the step.
891
526
 
892
- Use `@pypi_base` to set common packages required by all
527
+ Information in this decorator will augment any
528
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
529
+ you can use `@pypi_base` to set packages required by all
893
530
  steps and use `@pypi` to specify step-specific overrides.
894
531
 
532
+
895
533
  Parameters
896
534
  ----------
897
535
  packages : Dict[str, str], default: {}
898
- Packages to use for this flow. The key is the name of the package
536
+ Packages to use for this step. The key is the name of the package
899
537
  and the value is the version to use.
900
538
  python : str, optional, default: None
901
539
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
@@ -904,102 +542,402 @@ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[s
904
542
  ...
905
543
 
906
544
  @typing.overload
907
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
908
- ...
909
-
910
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
545
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
911
546
  """
912
- Specifies the PyPI packages for all steps of the flow.
547
+ Specifies the Conda environment for the step.
548
+
549
+ Information in this decorator will augment any
550
+ attributes set in the `@conda_base` flow-level decorator. Hence,
551
+ you can use `@conda_base` to set packages required by all
552
+ steps and use `@conda` to specify step-specific overrides.
913
553
 
914
- Use `@pypi_base` to set common packages required by all
915
- steps and use `@pypi` to specify step-specific overrides.
916
554
 
917
555
  Parameters
918
556
  ----------
919
- packages : Dict[str, str], default: {}
920
- Packages to use for this flow. The key is the name of the package
557
+ packages : Dict[str, str], default {}
558
+ Packages to use for this step. The key is the name of the package
921
559
  and the value is the version to use.
922
- python : str, optional, default: None
560
+ libraries : Dict[str, str], default {}
561
+ Supported for backward compatibility. When used with packages, packages will take precedence.
562
+ python : str, optional, default None
923
563
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
924
564
  that the version used will correspond to the version of the Python interpreter used to start the run.
565
+ disabled : bool, default False
566
+ If set to True, disables @conda.
925
567
  """
926
568
  ...
927
569
 
928
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
570
+ @typing.overload
571
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
572
+ ...
573
+
574
+ @typing.overload
575
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
576
+ ...
577
+
578
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
929
579
  """
930
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
931
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
580
+ Specifies the Conda environment for the step.
581
+
582
+ Information in this decorator will augment any
583
+ attributes set in the `@conda_base` flow-level decorator. Hence,
584
+ you can use `@conda_base` to set packages required by all
585
+ steps and use `@conda` to specify step-specific overrides.
932
586
 
933
587
 
934
588
  Parameters
935
589
  ----------
936
- timeout : int
937
- Time, in seconds before the task times out and fails. (Default: 3600)
938
- poke_interval : int
939
- Time in seconds that the job should wait in between each try. (Default: 60)
940
- mode : str
941
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
942
- exponential_backoff : bool
943
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
944
- pool : str
945
- the slot pool this task should run in,
946
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
947
- soft_fail : bool
948
- Set to true to mark the task as SKIPPED on failure. (Default: False)
949
- name : str
950
- Name of the sensor on Airflow
951
- description : str
952
- Description of sensor in the Airflow UI
953
- external_dag_id : str
954
- The dag_id that contains the task you want to wait for.
955
- external_task_ids : List[str]
956
- The list of task_ids that you want to wait for.
957
- If None (default value) the sensor waits for the DAG. (Default: None)
958
- allowed_states : List[str]
959
- Iterable of allowed states, (Default: ['success'])
960
- failed_states : List[str]
961
- Iterable of failed or dis-allowed states. (Default: None)
962
- execution_delta : datetime.timedelta
963
- time difference with the previous execution to look at,
964
- the default is the same logical date as the current task or DAG. (Default: None)
965
- check_existence: bool
966
- Set to True to check if the external task exists or check if
967
- the DAG to wait for exists. (Default: True)
590
+ packages : Dict[str, str], default {}
591
+ Packages to use for this step. The key is the name of the package
592
+ and the value is the version to use.
593
+ libraries : Dict[str, str], default {}
594
+ Supported for backward compatibility. When used with packages, packages will take precedence.
595
+ python : str, optional, default None
596
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
597
+ that the version used will correspond to the version of the Python interpreter used to start the run.
598
+ disabled : bool, default False
599
+ If set to True, disables @conda.
968
600
  """
969
601
  ...
970
602
 
971
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
603
+ @typing.overload
604
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
972
605
  """
973
- Specifies what flows belong to the same project.
606
+ Specifies environment variables to be set prior to the execution of a step.
974
607
 
975
- A project-specific namespace is created for all flows that
976
- use the same `@project(name)`.
608
+
609
+ Parameters
610
+ ----------
611
+ vars : Dict[str, str], default {}
612
+ Dictionary of environment variables to set.
613
+ """
614
+ ...
615
+
616
+ @typing.overload
617
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
618
+ ...
619
+
620
+ @typing.overload
621
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
622
+ ...
623
+
624
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
625
+ """
626
+ Specifies environment variables to be set prior to the execution of a step.
977
627
 
978
628
 
979
629
  Parameters
980
630
  ----------
981
- name : str
982
- Project name. Make sure that the name is unique amongst all
983
- projects that use the same production scheduler. The name may
984
- contain only lowercase alphanumeric characters and underscores.
631
+ vars : Dict[str, str], default {}
632
+ Dictionary of environment variables to set.
633
+ """
634
+ ...
635
+
636
+ @typing.overload
637
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
638
+ """
639
+ Decorator prototype for all step decorators. This function gets specialized
640
+ and imported for all decorators types by _import_plugin_decorators().
641
+ """
642
+ ...
643
+
644
+ @typing.overload
645
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
646
+ ...
647
+
648
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
649
+ """
650
+ Decorator prototype for all step decorators. This function gets specialized
651
+ and imported for all decorators types by _import_plugin_decorators().
652
+ """
653
+ ...
654
+
655
+ @typing.overload
656
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
657
+ """
658
+ Specifies that the step will success under all circumstances.
985
659
 
986
- branch : Optional[str], default None
987
- The branch to use. If not specified, the branch is set to
988
- `user.<username>` unless `production` is set to `True`. This can
989
- also be set on the command line using `--branch` as a top-level option.
990
- It is an error to specify `branch` in the decorator and on the command line.
660
+ The decorator will create an optional artifact, specified by `var`, which
661
+ contains the exception raised. You can use it to detect the presence
662
+ of errors, indicating that all happy-path artifacts produced by the step
663
+ are missing.
991
664
 
992
- production : bool, default False
993
- Whether or not the branch is the production branch. This can also be set on the
994
- command line using `--production` as a top-level option. It is an error to specify
995
- `production` in the decorator and on the command line.
996
- The project branch name will be:
997
- - if `branch` is specified:
998
- - if `production` is True: `prod.<branch>`
999
- - if `production` is False: `test.<branch>`
1000
- - if `branch` is not specified:
1001
- - if `production` is True: `prod`
1002
- - if `production` is False: `user.<username>`
665
+
666
+ Parameters
667
+ ----------
668
+ var : str, optional, default None
669
+ Name of the artifact in which to store the caught exception.
670
+ If not specified, the exception is not stored.
671
+ print_exception : bool, default True
672
+ Determines whether or not the exception is printed to
673
+ stdout when caught.
674
+ """
675
+ ...
676
+
677
+ @typing.overload
678
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
679
+ ...
680
+
681
+ @typing.overload
682
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
683
+ ...
684
+
685
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
686
+ """
687
+ Specifies that the step will success under all circumstances.
688
+
689
+ The decorator will create an optional artifact, specified by `var`, which
690
+ contains the exception raised. You can use it to detect the presence
691
+ of errors, indicating that all happy-path artifacts produced by the step
692
+ are missing.
693
+
694
+
695
+ Parameters
696
+ ----------
697
+ var : str, optional, default None
698
+ Name of the artifact in which to store the caught exception.
699
+ If not specified, the exception is not stored.
700
+ print_exception : bool, default True
701
+ Determines whether or not the exception is printed to
702
+ stdout when caught.
703
+ """
704
+ ...
705
+
706
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
707
+ """
708
+ Specifies that this step should execute on Kubernetes.
709
+
710
+
711
+ Parameters
712
+ ----------
713
+ cpu : int, default 1
714
+ Number of CPUs required for this step. If `@resources` is
715
+ also present, the maximum value from all decorators is used.
716
+ memory : int, default 4096
717
+ Memory size (in MB) required for this step. If
718
+ `@resources` is also present, the maximum value from all decorators is
719
+ used.
720
+ disk : int, default 10240
721
+ Disk size (in MB) required for this step. If
722
+ `@resources` is also present, the maximum value from all decorators is
723
+ used.
724
+ image : str, optional, default None
725
+ Docker image to use when launching on Kubernetes. If not specified, and
726
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
727
+ not, a default Docker image mapping to the current version of Python is used.
728
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
729
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
730
+ image_pull_secrets: List[str], default []
731
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
732
+ Kubernetes image pull secrets to use when pulling container images
733
+ in Kubernetes.
734
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
735
+ Kubernetes service account to use when launching pod in Kubernetes.
736
+ secrets : List[str], optional, default None
737
+ Kubernetes secrets to use when launching pod in Kubernetes. These
738
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
739
+ in Metaflow configuration.
740
+ node_selector: Union[Dict[str,str], str], optional, default None
741
+ Kubernetes node selector(s) to apply to the pod running the task.
742
+ Can be passed in as a comma separated string of values e.g.
743
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
744
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
745
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
746
+ Kubernetes namespace to use when launching pod in Kubernetes.
747
+ gpu : int, optional, default None
748
+ Number of GPUs required for this step. A value of zero implies that
749
+ the scheduled node should not have GPUs.
750
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
751
+ The vendor of the GPUs to be used for this step.
752
+ tolerations : List[Dict[str,str]], default []
753
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
754
+ Kubernetes tolerations to use when launching pod in Kubernetes.
755
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
756
+ Kubernetes labels to use when launching pod in Kubernetes.
757
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
758
+ Kubernetes annotations to use when launching pod in Kubernetes.
759
+ use_tmpfs : bool, default False
760
+ This enables an explicit tmpfs mount for this step.
761
+ tmpfs_tempdir : bool, default True
762
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
763
+ tmpfs_size : int, optional, default: None
764
+ The value for the size (in MiB) of the tmpfs mount for this step.
765
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
766
+ memory allocated for this step.
767
+ tmpfs_path : str, optional, default /metaflow_temp
768
+ Path to tmpfs mount for this step.
769
+ persistent_volume_claims : Dict[str, str], optional, default None
770
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
771
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
772
+ shared_memory: int, optional
773
+ Shared memory size (in MiB) required for this step
774
+ port: int, optional
775
+ Port number to specify in the Kubernetes job object
776
+ compute_pool : str, optional, default None
777
+ Compute pool to be used for for this step.
778
+ If not specified, any accessible compute pool within the perimeter is used.
779
+ hostname_resolution_timeout: int, default 10 * 60
780
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
781
+ Only applicable when @parallel is used.
782
+ qos: str, default: Burstable
783
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
784
+
785
+ security_context: Dict[str, Any], optional, default None
786
+ Container security context. Applies to the task container. Allows the following keys:
787
+ - privileged: bool, optional, default None
788
+ - allow_privilege_escalation: bool, optional, default None
789
+ - run_as_user: int, optional, default None
790
+ - run_as_group: int, optional, default None
791
+ - run_as_non_root: bool, optional, default None
792
+ """
793
+ ...
794
+
795
+ @typing.overload
796
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
797
+ """
798
+ Specifies a timeout for your step.
799
+
800
+ This decorator is useful if this step may hang indefinitely.
801
+
802
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
803
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
804
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
805
+
806
+ Note that all the values specified in parameters are added together so if you specify
807
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
808
+
809
+
810
+ Parameters
811
+ ----------
812
+ seconds : int, default 0
813
+ Number of seconds to wait prior to timing out.
814
+ minutes : int, default 0
815
+ Number of minutes to wait prior to timing out.
816
+ hours : int, default 0
817
+ Number of hours to wait prior to timing out.
818
+ """
819
+ ...
820
+
821
+ @typing.overload
822
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
823
+ ...
824
+
825
+ @typing.overload
826
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
827
+ ...
828
+
829
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
830
+ """
831
+ Specifies a timeout for your step.
832
+
833
+ This decorator is useful if this step may hang indefinitely.
834
+
835
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
836
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
837
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
838
+
839
+ Note that all the values specified in parameters are added together so if you specify
840
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
841
+
842
+
843
+ Parameters
844
+ ----------
845
+ seconds : int, default 0
846
+ Number of seconds to wait prior to timing out.
847
+ minutes : int, default 0
848
+ Number of minutes to wait prior to timing out.
849
+ hours : int, default 0
850
+ Number of hours to wait prior to timing out.
851
+ """
852
+ ...
853
+
854
+ @typing.overload
855
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
856
+ """
857
+ Specifies secrets to be retrieved and injected as environment variables prior to
858
+ the execution of a step.
859
+
860
+
861
+ Parameters
862
+ ----------
863
+ sources : List[Union[str, Dict[str, Any]]], default: []
864
+ List of secret specs, defining how the secrets are to be retrieved
865
+ role : str, optional, default: None
866
+ Role to use for fetching secrets
867
+ """
868
+ ...
869
+
870
+ @typing.overload
871
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
872
+ ...
873
+
874
+ @typing.overload
875
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
876
+ ...
877
+
878
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
879
+ """
880
+ Specifies secrets to be retrieved and injected as environment variables prior to
881
+ the execution of a step.
882
+
883
+
884
+ Parameters
885
+ ----------
886
+ sources : List[Union[str, Dict[str, Any]]], default: []
887
+ List of secret specs, defining how the secrets are to be retrieved
888
+ role : str, optional, default: None
889
+ Role to use for fetching secrets
890
+ """
891
+ ...
892
+
893
+ @typing.overload
894
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
895
+ """
896
+ Specifies the times when the flow should be run when running on a
897
+ production scheduler.
898
+
899
+
900
+ Parameters
901
+ ----------
902
+ hourly : bool, default False
903
+ Run the workflow hourly.
904
+ daily : bool, default True
905
+ Run the workflow daily.
906
+ weekly : bool, default False
907
+ Run the workflow weekly.
908
+ cron : str, optional, default None
909
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
910
+ specified by this expression.
911
+ timezone : str, optional, default None
912
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
913
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
914
+ """
915
+ ...
916
+
917
+ @typing.overload
918
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
919
+ ...
920
+
921
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
922
+ """
923
+ Specifies the times when the flow should be run when running on a
924
+ production scheduler.
925
+
926
+
927
+ Parameters
928
+ ----------
929
+ hourly : bool, default False
930
+ Run the workflow hourly.
931
+ daily : bool, default True
932
+ Run the workflow daily.
933
+ weekly : bool, default False
934
+ Run the workflow weekly.
935
+ cron : str, optional, default None
936
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
937
+ specified by this expression.
938
+ timezone : str, optional, default None
939
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
940
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1003
941
  """
1004
942
  ...
1005
943
 
@@ -1097,53 +1035,172 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1097
1035
  ...
1098
1036
 
1099
1037
  @typing.overload
1100
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1038
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1101
1039
  """
1102
- Specifies the times when the flow should be run when running on a
1103
- production scheduler.
1040
+ Specifies the Conda environment for all steps of the flow.
1041
+
1042
+ Use `@conda_base` to set common libraries required by all
1043
+ steps and use `@conda` to specify step-specific additions.
1104
1044
 
1105
1045
 
1106
1046
  Parameters
1107
1047
  ----------
1108
- hourly : bool, default False
1109
- Run the workflow hourly.
1110
- daily : bool, default True
1111
- Run the workflow daily.
1112
- weekly : bool, default False
1113
- Run the workflow weekly.
1114
- cron : str, optional, default None
1115
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1116
- specified by this expression.
1117
- timezone : str, optional, default None
1118
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1119
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1048
+ packages : Dict[str, str], default {}
1049
+ Packages to use for this flow. The key is the name of the package
1050
+ and the value is the version to use.
1051
+ libraries : Dict[str, str], default {}
1052
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1053
+ python : str, optional, default None
1054
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1055
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1056
+ disabled : bool, default False
1057
+ If set to True, disables Conda.
1120
1058
  """
1121
1059
  ...
1122
1060
 
1123
1061
  @typing.overload
1124
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1062
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1125
1063
  ...
1126
1064
 
1127
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1065
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1128
1066
  """
1129
- Specifies the times when the flow should be run when running on a
1130
- production scheduler.
1067
+ Specifies the Conda environment for all steps of the flow.
1068
+
1069
+ Use `@conda_base` to set common libraries required by all
1070
+ steps and use `@conda` to specify step-specific additions.
1131
1071
 
1132
1072
 
1133
1073
  Parameters
1134
1074
  ----------
1135
- hourly : bool, default False
1136
- Run the workflow hourly.
1137
- daily : bool, default True
1138
- Run the workflow daily.
1139
- weekly : bool, default False
1140
- Run the workflow weekly.
1141
- cron : str, optional, default None
1142
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1143
- specified by this expression.
1144
- timezone : str, optional, default None
1145
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1146
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1075
+ packages : Dict[str, str], default {}
1076
+ Packages to use for this flow. The key is the name of the package
1077
+ and the value is the version to use.
1078
+ libraries : Dict[str, str], default {}
1079
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1080
+ python : str, optional, default None
1081
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1082
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1083
+ disabled : bool, default False
1084
+ If set to True, disables Conda.
1085
+ """
1086
+ ...
1087
+
1088
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1089
+ """
1090
+ Specifies what flows belong to the same project.
1091
+
1092
+ A project-specific namespace is created for all flows that
1093
+ use the same `@project(name)`.
1094
+
1095
+
1096
+ Parameters
1097
+ ----------
1098
+ name : str
1099
+ Project name. Make sure that the name is unique amongst all
1100
+ projects that use the same production scheduler. The name may
1101
+ contain only lowercase alphanumeric characters and underscores.
1102
+
1103
+ branch : Optional[str], default None
1104
+ The branch to use. If not specified, the branch is set to
1105
+ `user.<username>` unless `production` is set to `True`. This can
1106
+ also be set on the command line using `--branch` as a top-level option.
1107
+ It is an error to specify `branch` in the decorator and on the command line.
1108
+
1109
+ production : bool, default False
1110
+ Whether or not the branch is the production branch. This can also be set on the
1111
+ command line using `--production` as a top-level option. It is an error to specify
1112
+ `production` in the decorator and on the command line.
1113
+ The project branch name will be:
1114
+ - if `branch` is specified:
1115
+ - if `production` is True: `prod.<branch>`
1116
+ - if `production` is False: `test.<branch>`
1117
+ - if `branch` is not specified:
1118
+ - if `production` is True: `prod`
1119
+ - if `production` is False: `user.<username>`
1120
+ """
1121
+ ...
1122
+
1123
+ @typing.overload
1124
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1125
+ """
1126
+ Specifies the PyPI packages for all steps of the flow.
1127
+
1128
+ Use `@pypi_base` to set common packages required by all
1129
+ steps and use `@pypi` to specify step-specific overrides.
1130
+
1131
+ Parameters
1132
+ ----------
1133
+ packages : Dict[str, str], default: {}
1134
+ Packages to use for this flow. The key is the name of the package
1135
+ and the value is the version to use.
1136
+ python : str, optional, default: None
1137
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1138
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1139
+ """
1140
+ ...
1141
+
1142
+ @typing.overload
1143
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1144
+ ...
1145
+
1146
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1147
+ """
1148
+ Specifies the PyPI packages for all steps of the flow.
1149
+
1150
+ Use `@pypi_base` to set common packages required by all
1151
+ steps and use `@pypi` to specify step-specific overrides.
1152
+
1153
+ Parameters
1154
+ ----------
1155
+ packages : Dict[str, str], default: {}
1156
+ Packages to use for this flow. The key is the name of the package
1157
+ and the value is the version to use.
1158
+ python : str, optional, default: None
1159
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1160
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1161
+ """
1162
+ ...
1163
+
1164
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1165
+ """
1166
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1167
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1168
+
1169
+
1170
+ Parameters
1171
+ ----------
1172
+ timeout : int
1173
+ Time, in seconds before the task times out and fails. (Default: 3600)
1174
+ poke_interval : int
1175
+ Time in seconds that the job should wait in between each try. (Default: 60)
1176
+ mode : str
1177
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1178
+ exponential_backoff : bool
1179
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1180
+ pool : str
1181
+ the slot pool this task should run in,
1182
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1183
+ soft_fail : bool
1184
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1185
+ name : str
1186
+ Name of the sensor on Airflow
1187
+ description : str
1188
+ Description of sensor in the Airflow UI
1189
+ external_dag_id : str
1190
+ The dag_id that contains the task you want to wait for.
1191
+ external_task_ids : List[str]
1192
+ The list of task_ids that you want to wait for.
1193
+ If None (default value) the sensor waits for the DAG. (Default: None)
1194
+ allowed_states : List[str]
1195
+ Iterable of allowed states, (Default: ['success'])
1196
+ failed_states : List[str]
1197
+ Iterable of failed or dis-allowed states. (Default: None)
1198
+ execution_delta : datetime.timedelta
1199
+ time difference with the previous execution to look at,
1200
+ the default is the same logical date as the current task or DAG. (Default: None)
1201
+ check_existence: bool
1202
+ Set to True to check if the external task exists or check if
1203
+ the DAG to wait for exists. (Default: True)
1147
1204
  """
1148
1205
  ...
1149
1206
 
@@ -1190,57 +1247,6 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1190
1247
  """
1191
1248
  ...
1192
1249
 
1193
- @typing.overload
1194
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1195
- """
1196
- Specifies the Conda environment for all steps of the flow.
1197
-
1198
- Use `@conda_base` to set common libraries required by all
1199
- steps and use `@conda` to specify step-specific additions.
1200
-
1201
-
1202
- Parameters
1203
- ----------
1204
- packages : Dict[str, str], default {}
1205
- Packages to use for this flow. The key is the name of the package
1206
- and the value is the version to use.
1207
- libraries : Dict[str, str], default {}
1208
- Supported for backward compatibility. When used with packages, packages will take precedence.
1209
- python : str, optional, default None
1210
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1211
- that the version used will correspond to the version of the Python interpreter used to start the run.
1212
- disabled : bool, default False
1213
- If set to True, disables Conda.
1214
- """
1215
- ...
1216
-
1217
- @typing.overload
1218
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1219
- ...
1220
-
1221
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1222
- """
1223
- Specifies the Conda environment for all steps of the flow.
1224
-
1225
- Use `@conda_base` to set common libraries required by all
1226
- steps and use `@conda` to specify step-specific additions.
1227
-
1228
-
1229
- Parameters
1230
- ----------
1231
- packages : Dict[str, str], default {}
1232
- Packages to use for this flow. The key is the name of the package
1233
- and the value is the version to use.
1234
- libraries : Dict[str, str], default {}
1235
- Supported for backward compatibility. When used with packages, packages will take precedence.
1236
- python : str, optional, default None
1237
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1238
- that the version used will correspond to the version of the Python interpreter used to start the run.
1239
- disabled : bool, default False
1240
- If set to True, disables Conda.
1241
- """
1242
- ...
1243
-
1244
1250
  @typing.overload
1245
1251
  def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1246
1252
  """