metaflow-stubs 2.14.0__py2.py3-none-any.whl → 2.14.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. metaflow-stubs/__init__.pyi +464 -464
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +93 -4
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +2 -2
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +26 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -4
  20. metaflow-stubs/metaflow_current.pyi +28 -28
  21. metaflow-stubs/multicore_utils.pyi +2 -2
  22. metaflow-stubs/parameters.pyi +2 -2
  23. metaflow-stubs/plugins/__init__.pyi +10 -10
  24. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  31. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  32. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  35. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  37. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  38. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  39. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  47. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  48. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  57. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  61. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  62. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  63. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  64. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  73. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  77. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  78. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  79. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  80. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  82. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  83. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  84. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  85. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  86. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  87. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  88. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  89. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  90. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  94. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  95. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  96. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  97. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  98. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  100. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  101. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  102. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  106. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  107. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  109. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/pypi/conda_environment.pyi +6 -6
  111. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  112. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  113. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  114. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  117. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  118. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  119. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  120. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  121. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  122. metaflow-stubs/pylint_wrapper.pyi +2 -2
  123. metaflow-stubs/runner/__init__.pyi +2 -2
  124. metaflow-stubs/runner/deployer.pyi +4 -4
  125. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  126. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  127. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  128. metaflow-stubs/runner/nbrun.pyi +2 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  130. metaflow-stubs/runner/utils.pyi +3 -3
  131. metaflow-stubs/system/__init__.pyi +2 -2
  132. metaflow-stubs/system/system_logger.pyi +2 -2
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. metaflow-stubs/user_configs/__init__.pyi +2 -2
  137. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  138. metaflow-stubs/user_configs/config_options.pyi +2 -2
  139. metaflow-stubs/user_configs/config_parameters.pyi +7 -7
  140. metaflow-stubs/version.pyi +2 -2
  141. {metaflow_stubs-2.14.0.dist-info → metaflow_stubs-2.14.1.dist-info}/METADATA +2 -2
  142. metaflow_stubs-2.14.1.dist-info/RECORD +145 -0
  143. metaflow_stubs-2.14.0.dist-info/RECORD +0 -145
  144. {metaflow_stubs-2.14.0.dist-info → metaflow_stubs-2.14.1.dist-info}/WHEEL +0 -0
  145. {metaflow_stubs-2.14.0.dist-info → metaflow_stubs-2.14.1.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.14.0 #
4
- # Generated on 2025-02-11T19:31:25.349322 #
3
+ # MF version: 2.14.1 #
4
+ # Generated on 2025-02-21T00:53:08.401724 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -143,186 +143,102 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
143
143
  ...
144
144
 
145
145
  @typing.overload
146
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
147
- """
148
- Specifies the number of times the task corresponding
149
- to a step needs to be retried.
150
-
151
- This decorator is useful for handling transient errors, such as networking issues.
152
- If your task contains operations that can't be retried safely, e.g. database updates,
153
- it is advisable to annotate it with `@retry(times=0)`.
154
-
155
- This can be used in conjunction with the `@catch` decorator. The `@catch`
156
- decorator will execute a no-op task after all retries have been exhausted,
157
- ensuring that the flow execution can continue.
158
-
159
-
160
- Parameters
161
- ----------
162
- times : int, default 3
163
- Number of times to retry this task.
164
- minutes_between_retries : int, default 2
165
- Number of minutes between retries.
166
- """
167
- ...
168
-
169
- @typing.overload
170
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
171
- ...
172
-
173
- @typing.overload
174
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
175
- ...
176
-
177
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
178
- """
179
- Specifies the number of times the task corresponding
180
- to a step needs to be retried.
181
-
182
- This decorator is useful for handling transient errors, such as networking issues.
183
- If your task contains operations that can't be retried safely, e.g. database updates,
184
- it is advisable to annotate it with `@retry(times=0)`.
185
-
186
- This can be used in conjunction with the `@catch` decorator. The `@catch`
187
- decorator will execute a no-op task after all retries have been exhausted,
188
- ensuring that the flow execution can continue.
189
-
190
-
191
- Parameters
192
- ----------
193
- times : int, default 3
194
- Number of times to retry this task.
195
- minutes_between_retries : int, default 2
196
- Number of minutes between retries.
197
- """
198
- ...
199
-
200
- @typing.overload
201
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
146
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
202
147
  """
203
- Specifies the Conda environment for the step.
148
+ Specifies the PyPI packages for the step.
204
149
 
205
150
  Information in this decorator will augment any
206
- attributes set in the `@conda_base` flow-level decorator. Hence,
207
- you can use `@conda_base` to set packages required by all
208
- steps and use `@conda` to specify step-specific overrides.
151
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
152
+ you can use `@pypi_base` to set packages required by all
153
+ steps and use `@pypi` to specify step-specific overrides.
209
154
 
210
155
 
211
156
  Parameters
212
157
  ----------
213
- packages : Dict[str, str], default {}
158
+ packages : Dict[str, str], default: {}
214
159
  Packages to use for this step. The key is the name of the package
215
160
  and the value is the version to use.
216
- libraries : Dict[str, str], default {}
217
- Supported for backward compatibility. When used with packages, packages will take precedence.
218
- python : str, optional, default None
161
+ python : str, optional, default: None
219
162
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
220
163
  that the version used will correspond to the version of the Python interpreter used to start the run.
221
- disabled : bool, default False
222
- If set to True, disables @conda.
223
164
  """
224
165
  ...
225
166
 
226
167
  @typing.overload
227
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
168
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
228
169
  ...
229
170
 
230
171
  @typing.overload
231
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
172
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
232
173
  ...
233
174
 
234
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
175
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
235
176
  """
236
- Specifies the Conda environment for the step.
177
+ Specifies the PyPI packages for the step.
237
178
 
238
179
  Information in this decorator will augment any
239
- attributes set in the `@conda_base` flow-level decorator. Hence,
240
- you can use `@conda_base` to set packages required by all
241
- steps and use `@conda` to specify step-specific overrides.
180
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
181
+ you can use `@pypi_base` to set packages required by all
182
+ steps and use `@pypi` to specify step-specific overrides.
242
183
 
243
184
 
244
185
  Parameters
245
186
  ----------
246
- packages : Dict[str, str], default {}
187
+ packages : Dict[str, str], default: {}
247
188
  Packages to use for this step. The key is the name of the package
248
189
  and the value is the version to use.
249
- libraries : Dict[str, str], default {}
250
- Supported for backward compatibility. When used with packages, packages will take precedence.
251
- python : str, optional, default None
190
+ python : str, optional, default: None
252
191
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
253
192
  that the version used will correspond to the version of the Python interpreter used to start the run.
254
- disabled : bool, default False
255
- If set to True, disables @conda.
256
- """
257
- ...
258
-
259
- @typing.overload
260
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
261
- """
262
- Parameters
263
- ----------
264
193
  """
265
194
  ...
266
195
 
267
196
  @typing.overload
268
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
269
- ...
270
-
271
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
272
- """
273
- Parameters
274
- ----------
275
- """
276
- ...
277
-
278
- @typing.overload
279
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
197
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
280
198
  """
281
- Specifies the PyPI packages for the step.
199
+ Creates a human-readable report, a Metaflow Card, after this step completes.
282
200
 
283
- Information in this decorator will augment any
284
- attributes set in the `@pyi_base` flow-level decorator. Hence,
285
- you can use `@pypi_base` to set packages required by all
286
- steps and use `@pypi` to specify step-specific overrides.
201
+ Note that you may add multiple `@card` decorators in a step with different parameters.
287
202
 
288
203
 
289
204
  Parameters
290
205
  ----------
291
- packages : Dict[str, str], default: {}
292
- Packages to use for this step. The key is the name of the package
293
- and the value is the version to use.
294
- python : str, optional, default: None
295
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
296
- that the version used will correspond to the version of the Python interpreter used to start the run.
206
+ type : str, default 'default'
207
+ Card type.
208
+ id : str, optional, default None
209
+ If multiple cards are present, use this id to identify this card.
210
+ options : Dict[str, Any], default {}
211
+ Options passed to the card. The contents depend on the card type.
212
+ timeout : int, default 45
213
+ Interrupt reporting if it takes more than this many seconds.
297
214
  """
298
215
  ...
299
216
 
300
217
  @typing.overload
301
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
218
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
302
219
  ...
303
220
 
304
221
  @typing.overload
305
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
222
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
306
223
  ...
307
224
 
308
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
225
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
309
226
  """
310
- Specifies the PyPI packages for the step.
227
+ Creates a human-readable report, a Metaflow Card, after this step completes.
311
228
 
312
- Information in this decorator will augment any
313
- attributes set in the `@pyi_base` flow-level decorator. Hence,
314
- you can use `@pypi_base` to set packages required by all
315
- steps and use `@pypi` to specify step-specific overrides.
229
+ Note that you may add multiple `@card` decorators in a step with different parameters.
316
230
 
317
231
 
318
232
  Parameters
319
233
  ----------
320
- packages : Dict[str, str], default: {}
321
- Packages to use for this step. The key is the name of the package
322
- and the value is the version to use.
323
- python : str, optional, default: None
324
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
325
- that the version used will correspond to the version of the Python interpreter used to start the run.
234
+ type : str, default 'default'
235
+ Card type.
236
+ id : str, optional, default None
237
+ If multiple cards are present, use this id to identify this card.
238
+ options : Dict[str, Any], default {}
239
+ Options passed to the card. The contents depend on the card type.
240
+ timeout : int, default 45
241
+ Interrupt reporting if it takes more than this many seconds.
326
242
  """
327
243
  ...
328
244
 
@@ -385,169 +301,6 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
385
301
  """
386
302
  ...
387
303
 
388
- @typing.overload
389
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
390
- """
391
- Specifies that the step will success under all circumstances.
392
-
393
- The decorator will create an optional artifact, specified by `var`, which
394
- contains the exception raised. You can use it to detect the presence
395
- of errors, indicating that all happy-path artifacts produced by the step
396
- are missing.
397
-
398
-
399
- Parameters
400
- ----------
401
- var : str, optional, default None
402
- Name of the artifact in which to store the caught exception.
403
- If not specified, the exception is not stored.
404
- print_exception : bool, default True
405
- Determines whether or not the exception is printed to
406
- stdout when caught.
407
- """
408
- ...
409
-
410
- @typing.overload
411
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
412
- ...
413
-
414
- @typing.overload
415
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
416
- ...
417
-
418
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
419
- """
420
- Specifies that the step will success under all circumstances.
421
-
422
- The decorator will create an optional artifact, specified by `var`, which
423
- contains the exception raised. You can use it to detect the presence
424
- of errors, indicating that all happy-path artifacts produced by the step
425
- are missing.
426
-
427
-
428
- Parameters
429
- ----------
430
- var : str, optional, default None
431
- Name of the artifact in which to store the caught exception.
432
- If not specified, the exception is not stored.
433
- print_exception : bool, default True
434
- Determines whether or not the exception is printed to
435
- stdout when caught.
436
- """
437
- ...
438
-
439
- @typing.overload
440
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
441
- """
442
- Specifies the resources needed when executing this step.
443
-
444
- Use `@resources` to specify the resource requirements
445
- independently of the specific compute layer (`@batch`, `@kubernetes`).
446
-
447
- You can choose the compute layer on the command line by executing e.g.
448
- ```
449
- python myflow.py run --with batch
450
- ```
451
- or
452
- ```
453
- python myflow.py run --with kubernetes
454
- ```
455
- which executes the flow on the desired system using the
456
- requirements specified in `@resources`.
457
-
458
-
459
- Parameters
460
- ----------
461
- cpu : int, default 1
462
- Number of CPUs required for this step.
463
- gpu : int, optional, default None
464
- Number of GPUs required for this step.
465
- disk : int, optional, default None
466
- Disk size (in MB) required for this step. Only applies on Kubernetes.
467
- memory : int, default 4096
468
- Memory size (in MB) required for this step.
469
- shared_memory : int, optional, default None
470
- The value for the size (in MiB) of the /dev/shm volume for this step.
471
- This parameter maps to the `--shm-size` option in Docker.
472
- """
473
- ...
474
-
475
- @typing.overload
476
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
477
- ...
478
-
479
- @typing.overload
480
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
481
- ...
482
-
483
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
484
- """
485
- Specifies the resources needed when executing this step.
486
-
487
- Use `@resources` to specify the resource requirements
488
- independently of the specific compute layer (`@batch`, `@kubernetes`).
489
-
490
- You can choose the compute layer on the command line by executing e.g.
491
- ```
492
- python myflow.py run --with batch
493
- ```
494
- or
495
- ```
496
- python myflow.py run --with kubernetes
497
- ```
498
- which executes the flow on the desired system using the
499
- requirements specified in `@resources`.
500
-
501
-
502
- Parameters
503
- ----------
504
- cpu : int, default 1
505
- Number of CPUs required for this step.
506
- gpu : int, optional, default None
507
- Number of GPUs required for this step.
508
- disk : int, optional, default None
509
- Disk size (in MB) required for this step. Only applies on Kubernetes.
510
- memory : int, default 4096
511
- Memory size (in MB) required for this step.
512
- shared_memory : int, optional, default None
513
- The value for the size (in MiB) of the /dev/shm volume for this step.
514
- This parameter maps to the `--shm-size` option in Docker.
515
- """
516
- ...
517
-
518
- @typing.overload
519
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
520
- """
521
- Specifies environment variables to be set prior to the execution of a step.
522
-
523
-
524
- Parameters
525
- ----------
526
- vars : Dict[str, str], default {}
527
- Dictionary of environment variables to set.
528
- """
529
- ...
530
-
531
- @typing.overload
532
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
533
- ...
534
-
535
- @typing.overload
536
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
537
- ...
538
-
539
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
540
- """
541
- Specifies environment variables to be set prior to the execution of a step.
542
-
543
-
544
- Parameters
545
- ----------
546
- vars : Dict[str, str], default {}
547
- Dictionary of environment variables to set.
548
- """
549
- ...
550
-
551
304
  @typing.overload
552
305
  def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
553
306
  """
@@ -660,6 +413,136 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
660
413
  """
661
414
  ...
662
415
 
416
+ @typing.overload
417
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
418
+ """
419
+ Specifies the resources needed when executing this step.
420
+
421
+ Use `@resources` to specify the resource requirements
422
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
423
+
424
+ You can choose the compute layer on the command line by executing e.g.
425
+ ```
426
+ python myflow.py run --with batch
427
+ ```
428
+ or
429
+ ```
430
+ python myflow.py run --with kubernetes
431
+ ```
432
+ which executes the flow on the desired system using the
433
+ requirements specified in `@resources`.
434
+
435
+
436
+ Parameters
437
+ ----------
438
+ cpu : int, default 1
439
+ Number of CPUs required for this step.
440
+ gpu : int, optional, default None
441
+ Number of GPUs required for this step.
442
+ disk : int, optional, default None
443
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
444
+ memory : int, default 4096
445
+ Memory size (in MB) required for this step.
446
+ shared_memory : int, optional, default None
447
+ The value for the size (in MiB) of the /dev/shm volume for this step.
448
+ This parameter maps to the `--shm-size` option in Docker.
449
+ """
450
+ ...
451
+
452
+ @typing.overload
453
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
454
+ ...
455
+
456
+ @typing.overload
457
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
458
+ ...
459
+
460
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
461
+ """
462
+ Specifies the resources needed when executing this step.
463
+
464
+ Use `@resources` to specify the resource requirements
465
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
466
+
467
+ You can choose the compute layer on the command line by executing e.g.
468
+ ```
469
+ python myflow.py run --with batch
470
+ ```
471
+ or
472
+ ```
473
+ python myflow.py run --with kubernetes
474
+ ```
475
+ which executes the flow on the desired system using the
476
+ requirements specified in `@resources`.
477
+
478
+
479
+ Parameters
480
+ ----------
481
+ cpu : int, default 1
482
+ Number of CPUs required for this step.
483
+ gpu : int, optional, default None
484
+ Number of GPUs required for this step.
485
+ disk : int, optional, default None
486
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
487
+ memory : int, default 4096
488
+ Memory size (in MB) required for this step.
489
+ shared_memory : int, optional, default None
490
+ The value for the size (in MiB) of the /dev/shm volume for this step.
491
+ This parameter maps to the `--shm-size` option in Docker.
492
+ """
493
+ ...
494
+
495
+ @typing.overload
496
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
497
+ """
498
+ Specifies that the step will success under all circumstances.
499
+
500
+ The decorator will create an optional artifact, specified by `var`, which
501
+ contains the exception raised. You can use it to detect the presence
502
+ of errors, indicating that all happy-path artifacts produced by the step
503
+ are missing.
504
+
505
+
506
+ Parameters
507
+ ----------
508
+ var : str, optional, default None
509
+ Name of the artifact in which to store the caught exception.
510
+ If not specified, the exception is not stored.
511
+ print_exception : bool, default True
512
+ Determines whether or not the exception is printed to
513
+ stdout when caught.
514
+ """
515
+ ...
516
+
517
+ @typing.overload
518
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
519
+ ...
520
+
521
+ @typing.overload
522
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
523
+ ...
524
+
525
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
526
+ """
527
+ Specifies that the step will success under all circumstances.
528
+
529
+ The decorator will create an optional artifact, specified by `var`, which
530
+ contains the exception raised. You can use it to detect the presence
531
+ of errors, indicating that all happy-path artifacts produced by the step
532
+ are missing.
533
+
534
+
535
+ Parameters
536
+ ----------
537
+ var : str, optional, default None
538
+ Name of the artifact in which to store the caught exception.
539
+ If not specified, the exception is not stored.
540
+ print_exception : bool, default True
541
+ Determines whether or not the exception is printed to
542
+ stdout when caught.
543
+ """
544
+ ...
545
+
663
546
  @typing.overload
664
547
  def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
665
548
  """
@@ -810,92 +693,168 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
810
693
  ...
811
694
 
812
695
  @typing.overload
813
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
696
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
814
697
  """
815
- Creates a human-readable report, a Metaflow Card, after this step completes.
816
-
817
- Note that you may add multiple `@card` decorators in a step with different parameters.
698
+ Parameters
699
+ ----------
700
+ """
701
+ ...
702
+
703
+ @typing.overload
704
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
705
+ ...
706
+
707
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
708
+ """
709
+ Parameters
710
+ ----------
711
+ """
712
+ ...
713
+
714
+ @typing.overload
715
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
716
+ """
717
+ Specifies environment variables to be set prior to the execution of a step.
818
718
 
819
719
 
820
720
  Parameters
821
721
  ----------
822
- type : str, default 'default'
823
- Card type.
824
- id : str, optional, default None
825
- If multiple cards are present, use this id to identify this card.
826
- options : Dict[str, Any], default {}
827
- Options passed to the card. The contents depend on the card type.
828
- timeout : int, default 45
829
- Interrupt reporting if it takes more than this many seconds.
722
+ vars : Dict[str, str], default {}
723
+ Dictionary of environment variables to set.
830
724
  """
831
725
  ...
832
726
 
833
727
  @typing.overload
834
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
728
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
835
729
  ...
836
730
 
837
731
  @typing.overload
838
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
732
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
839
733
  ...
840
734
 
841
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
735
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
842
736
  """
843
- Creates a human-readable report, a Metaflow Card, after this step completes.
844
-
845
- Note that you may add multiple `@card` decorators in a step with different parameters.
737
+ Specifies environment variables to be set prior to the execution of a step.
846
738
 
847
739
 
848
740
  Parameters
849
741
  ----------
850
- type : str, default 'default'
851
- Card type.
852
- id : str, optional, default None
853
- If multiple cards are present, use this id to identify this card.
854
- options : Dict[str, Any], default {}
855
- Options passed to the card. The contents depend on the card type.
856
- timeout : int, default 45
857
- Interrupt reporting if it takes more than this many seconds.
742
+ vars : Dict[str, str], default {}
743
+ Dictionary of environment variables to set.
858
744
  """
859
745
  ...
860
746
 
861
747
  @typing.overload
862
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
748
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
863
749
  """
864
- Specifies the PyPI packages for all steps of the flow.
750
+ Specifies the Conda environment for the step.
751
+
752
+ Information in this decorator will augment any
753
+ attributes set in the `@conda_base` flow-level decorator. Hence,
754
+ you can use `@conda_base` to set packages required by all
755
+ steps and use `@conda` to specify step-specific overrides.
865
756
 
866
- Use `@pypi_base` to set common packages required by all
867
- steps and use `@pypi` to specify step-specific overrides.
868
757
 
869
758
  Parameters
870
759
  ----------
871
- packages : Dict[str, str], default: {}
872
- Packages to use for this flow. The key is the name of the package
760
+ packages : Dict[str, str], default {}
761
+ Packages to use for this step. The key is the name of the package
873
762
  and the value is the version to use.
874
- python : str, optional, default: None
763
+ libraries : Dict[str, str], default {}
764
+ Supported for backward compatibility. When used with packages, packages will take precedence.
765
+ python : str, optional, default None
875
766
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
876
767
  that the version used will correspond to the version of the Python interpreter used to start the run.
768
+ disabled : bool, default False
769
+ If set to True, disables @conda.
877
770
  """
878
771
  ...
879
772
 
880
773
  @typing.overload
881
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
774
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
882
775
  ...
883
776
 
884
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
777
+ @typing.overload
778
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
779
+ ...
780
+
781
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
885
782
  """
886
- Specifies the PyPI packages for all steps of the flow.
783
+ Specifies the Conda environment for the step.
784
+
785
+ Information in this decorator will augment any
786
+ attributes set in the `@conda_base` flow-level decorator. Hence,
787
+ you can use `@conda_base` to set packages required by all
788
+ steps and use `@conda` to specify step-specific overrides.
887
789
 
888
- Use `@pypi_base` to set common packages required by all
889
- steps and use `@pypi` to specify step-specific overrides.
890
790
 
891
791
  Parameters
892
792
  ----------
893
- packages : Dict[str, str], default: {}
894
- Packages to use for this flow. The key is the name of the package
793
+ packages : Dict[str, str], default {}
794
+ Packages to use for this step. The key is the name of the package
895
795
  and the value is the version to use.
896
- python : str, optional, default: None
796
+ libraries : Dict[str, str], default {}
797
+ Supported for backward compatibility. When used with packages, packages will take precedence.
798
+ python : str, optional, default None
897
799
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
898
800
  that the version used will correspond to the version of the Python interpreter used to start the run.
801
+ disabled : bool, default False
802
+ If set to True, disables @conda.
803
+ """
804
+ ...
805
+
806
+ @typing.overload
807
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
808
+ """
809
+ Specifies the number of times the task corresponding
810
+ to a step needs to be retried.
811
+
812
+ This decorator is useful for handling transient errors, such as networking issues.
813
+ If your task contains operations that can't be retried safely, e.g. database updates,
814
+ it is advisable to annotate it with `@retry(times=0)`.
815
+
816
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
817
+ decorator will execute a no-op task after all retries have been exhausted,
818
+ ensuring that the flow execution can continue.
819
+
820
+
821
+ Parameters
822
+ ----------
823
+ times : int, default 3
824
+ Number of times to retry this task.
825
+ minutes_between_retries : int, default 2
826
+ Number of minutes between retries.
827
+ """
828
+ ...
829
+
830
+ @typing.overload
831
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
832
+ ...
833
+
834
+ @typing.overload
835
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
836
+ ...
837
+
838
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
839
+ """
840
+ Specifies the number of times the task corresponding
841
+ to a step needs to be retried.
842
+
843
+ This decorator is useful for handling transient errors, such as networking issues.
844
+ If your task contains operations that can't be retried safely, e.g. database updates,
845
+ it is advisable to annotate it with `@retry(times=0)`.
846
+
847
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
848
+ decorator will execute a no-op task after all retries have been exhausted,
849
+ ensuring that the flow execution can continue.
850
+
851
+
852
+ Parameters
853
+ ----------
854
+ times : int, default 3
855
+ Number of times to retry this task.
856
+ minutes_between_retries : int, default 2
857
+ Number of minutes between retries.
899
858
  """
900
859
  ...
901
860
 
@@ -950,81 +909,96 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
950
909
  """
951
910
  ...
952
911
 
953
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
912
+ @typing.overload
913
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
954
914
  """
955
- Specifies what flows belong to the same project.
956
-
957
- A project-specific namespace is created for all flows that
958
- use the same `@project(name)`.
959
-
915
+ Specifies the event(s) that this flow depends on.
960
916
 
961
- Parameters
962
- ----------
963
- name : str
964
- Project name. Make sure that the name is unique amongst all
965
- projects that use the same production scheduler. The name may
966
- contain only lowercase alphanumeric characters and underscores.
917
+ ```
918
+ @trigger(event='foo')
919
+ ```
920
+ or
921
+ ```
922
+ @trigger(events=['foo', 'bar'])
923
+ ```
967
924
 
968
- branch : Optional[str], default None
969
- The branch to use. If not specified, the branch is set to
970
- `user.<username>` unless `production` is set to `True`. This can
971
- also be set on the command line using `--branch` as a top-level option.
972
- It is an error to specify `branch` in the decorator and on the command line.
925
+ Additionally, you can specify the parameter mappings
926
+ to map event payload to Metaflow parameters for the flow.
927
+ ```
928
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
929
+ ```
930
+ or
931
+ ```
932
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
933
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
934
+ ```
973
935
 
974
- production : bool, default False
975
- Whether or not the branch is the production branch. This can also be set on the
976
- command line using `--production` as a top-level option. It is an error to specify
977
- `production` in the decorator and on the command line.
978
- The project branch name will be:
979
- - if `branch` is specified:
980
- - if `production` is True: `prod.<branch>`
981
- - if `production` is False: `test.<branch>`
982
- - if `branch` is not specified:
983
- - if `production` is True: `prod`
984
- - if `production` is False: `user.<username>`
936
+ 'parameters' can also be a list of strings and tuples like so:
937
+ ```
938
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
939
+ ```
940
+ This is equivalent to:
941
+ ```
942
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
943
+ ```
944
+
945
+
946
+ Parameters
947
+ ----------
948
+ event : Union[str, Dict[str, Any]], optional, default None
949
+ Event dependency for this flow.
950
+ events : List[Union[str, Dict[str, Any]]], default []
951
+ Events dependency for this flow.
952
+ options : Dict[str, Any], default {}
953
+ Backend-specific configuration for tuning eventing behavior.
985
954
  """
986
955
  ...
987
956
 
988
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
957
+ @typing.overload
958
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
959
+ ...
960
+
961
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
989
962
  """
990
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
991
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
992
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
993
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
994
- starts only after all sensors finish.
963
+ Specifies the event(s) that this flow depends on.
964
+
965
+ ```
966
+ @trigger(event='foo')
967
+ ```
968
+ or
969
+ ```
970
+ @trigger(events=['foo', 'bar'])
971
+ ```
972
+
973
+ Additionally, you can specify the parameter mappings
974
+ to map event payload to Metaflow parameters for the flow.
975
+ ```
976
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
977
+ ```
978
+ or
979
+ ```
980
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
981
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
982
+ ```
983
+
984
+ 'parameters' can also be a list of strings and tuples like so:
985
+ ```
986
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
987
+ ```
988
+ This is equivalent to:
989
+ ```
990
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
991
+ ```
995
992
 
996
993
 
997
994
  Parameters
998
995
  ----------
999
- timeout : int
1000
- Time, in seconds before the task times out and fails. (Default: 3600)
1001
- poke_interval : int
1002
- Time in seconds that the job should wait in between each try. (Default: 60)
1003
- mode : str
1004
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1005
- exponential_backoff : bool
1006
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1007
- pool : str
1008
- the slot pool this task should run in,
1009
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1010
- soft_fail : bool
1011
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1012
- name : str
1013
- Name of the sensor on Airflow
1014
- description : str
1015
- Description of sensor in the Airflow UI
1016
- bucket_key : Union[str, List[str]]
1017
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1018
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1019
- bucket_name : str
1020
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1021
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1022
- wildcard_match : bool
1023
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1024
- aws_conn_id : str
1025
- a reference to the s3 connection on Airflow. (Default: None)
1026
- verify : bool
1027
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
996
+ event : Union[str, Dict[str, Any]], optional, default None
997
+ Event dependency for this flow.
998
+ events : List[Union[str, Dict[str, Any]]], default []
999
+ Events dependency for this flow.
1000
+ options : Dict[str, Any], default {}
1001
+ Backend-specific configuration for tuning eventing behavior.
1028
1002
  """
1029
1003
  ...
1030
1004
 
@@ -1180,6 +1154,49 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1180
1154
  """
1181
1155
  ...
1182
1156
 
1157
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1158
+ """
1159
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1160
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1161
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1162
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1163
+ starts only after all sensors finish.
1164
+
1165
+
1166
+ Parameters
1167
+ ----------
1168
+ timeout : int
1169
+ Time, in seconds before the task times out and fails. (Default: 3600)
1170
+ poke_interval : int
1171
+ Time in seconds that the job should wait in between each try. (Default: 60)
1172
+ mode : str
1173
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1174
+ exponential_backoff : bool
1175
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1176
+ pool : str
1177
+ the slot pool this task should run in,
1178
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1179
+ soft_fail : bool
1180
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1181
+ name : str
1182
+ Name of the sensor on Airflow
1183
+ description : str
1184
+ Description of sensor in the Airflow UI
1185
+ bucket_key : Union[str, List[str]]
1186
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1187
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1188
+ bucket_name : str
1189
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1190
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1191
+ wildcard_match : bool
1192
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1193
+ aws_conn_id : str
1194
+ a reference to the s3 connection on Airflow. (Default: None)
1195
+ verify : bool
1196
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1197
+ """
1198
+ ...
1199
+
1183
1200
  def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1184
1201
  """
1185
1202
  The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
@@ -1223,96 +1240,79 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1223
1240
  """
1224
1241
  ...
1225
1242
 
1226
- @typing.overload
1227
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1243
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1228
1244
  """
1229
- Specifies the event(s) that this flow depends on.
1245
+ Specifies what flows belong to the same project.
1230
1246
 
1231
- ```
1232
- @trigger(event='foo')
1233
- ```
1234
- or
1235
- ```
1236
- @trigger(events=['foo', 'bar'])
1237
- ```
1247
+ A project-specific namespace is created for all flows that
1248
+ use the same `@project(name)`.
1238
1249
 
1239
- Additionally, you can specify the parameter mappings
1240
- to map event payload to Metaflow parameters for the flow.
1241
- ```
1242
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1243
- ```
1244
- or
1245
- ```
1246
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1247
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1248
- ```
1249
1250
 
1250
- 'parameters' can also be a list of strings and tuples like so:
1251
- ```
1252
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1253
- ```
1254
- This is equivalent to:
1255
- ```
1256
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1257
- ```
1251
+ Parameters
1252
+ ----------
1253
+ name : str
1254
+ Project name. Make sure that the name is unique amongst all
1255
+ projects that use the same production scheduler. The name may
1256
+ contain only lowercase alphanumeric characters and underscores.
1257
+
1258
+ branch : Optional[str], default None
1259
+ The branch to use. If not specified, the branch is set to
1260
+ `user.<username>` unless `production` is set to `True`. This can
1261
+ also be set on the command line using `--branch` as a top-level option.
1262
+ It is an error to specify `branch` in the decorator and on the command line.
1263
+
1264
+ production : bool, default False
1265
+ Whether or not the branch is the production branch. This can also be set on the
1266
+ command line using `--production` as a top-level option. It is an error to specify
1267
+ `production` in the decorator and on the command line.
1268
+ The project branch name will be:
1269
+ - if `branch` is specified:
1270
+ - if `production` is True: `prod.<branch>`
1271
+ - if `production` is False: `test.<branch>`
1272
+ - if `branch` is not specified:
1273
+ - if `production` is True: `prod`
1274
+ - if `production` is False: `user.<username>`
1275
+ """
1276
+ ...
1277
+
1278
+ @typing.overload
1279
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1280
+ """
1281
+ Specifies the PyPI packages for all steps of the flow.
1258
1282
 
1283
+ Use `@pypi_base` to set common packages required by all
1284
+ steps and use `@pypi` to specify step-specific overrides.
1259
1285
 
1260
1286
  Parameters
1261
1287
  ----------
1262
- event : Union[str, Dict[str, Any]], optional, default None
1263
- Event dependency for this flow.
1264
- events : List[Union[str, Dict[str, Any]]], default []
1265
- Events dependency for this flow.
1266
- options : Dict[str, Any], default {}
1267
- Backend-specific configuration for tuning eventing behavior.
1288
+ packages : Dict[str, str], default: {}
1289
+ Packages to use for this flow. The key is the name of the package
1290
+ and the value is the version to use.
1291
+ python : str, optional, default: None
1292
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1293
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1268
1294
  """
1269
1295
  ...
1270
1296
 
1271
1297
  @typing.overload
1272
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1298
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1273
1299
  ...
1274
1300
 
1275
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1301
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1276
1302
  """
1277
- Specifies the event(s) that this flow depends on.
1278
-
1279
- ```
1280
- @trigger(event='foo')
1281
- ```
1282
- or
1283
- ```
1284
- @trigger(events=['foo', 'bar'])
1285
- ```
1286
-
1287
- Additionally, you can specify the parameter mappings
1288
- to map event payload to Metaflow parameters for the flow.
1289
- ```
1290
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1291
- ```
1292
- or
1293
- ```
1294
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1295
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1296
- ```
1297
-
1298
- 'parameters' can also be a list of strings and tuples like so:
1299
- ```
1300
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1301
- ```
1302
- This is equivalent to:
1303
- ```
1304
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1305
- ```
1303
+ Specifies the PyPI packages for all steps of the flow.
1306
1304
 
1305
+ Use `@pypi_base` to set common packages required by all
1306
+ steps and use `@pypi` to specify step-specific overrides.
1307
1307
 
1308
1308
  Parameters
1309
1309
  ----------
1310
- event : Union[str, Dict[str, Any]], optional, default None
1311
- Event dependency for this flow.
1312
- events : List[Union[str, Dict[str, Any]]], default []
1313
- Events dependency for this flow.
1314
- options : Dict[str, Any], default {}
1315
- Backend-specific configuration for tuning eventing behavior.
1310
+ packages : Dict[str, str], default: {}
1311
+ Packages to use for this flow. The key is the name of the package
1312
+ and the value is the version to use.
1313
+ python : str, optional, default: None
1314
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1315
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1316
1316
  """
1317
1317
  ...
1318
1318