ob-metaflow-stubs 6.0.3.145__py2.py3-none-any.whl → 6.0.3.146__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. metaflow-stubs/__init__.pyi +555 -555
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +99 -99
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +4 -4
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +5 -5
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +5 -5
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  61. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  62. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  63. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  74. metaflow-stubs/multicore_utils.pyi +2 -2
  75. metaflow-stubs/parameters.pyi +4 -4
  76. metaflow-stubs/plugins/__init__.pyi +13 -13
  77. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  79. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  80. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  82. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  83. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  84. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  85. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  86. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  87. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  88. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  89. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +5 -5
  90. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  91. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  92. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  93. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  94. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  96. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  97. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  100. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  102. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  103. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  104. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  105. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +5 -5
  106. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  107. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  108. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  109. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  110. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  111. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  112. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  113. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  114. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  116. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  117. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  118. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  119. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  120. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  121. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_modules/components.pyi +5 -5
  123. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  124. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  125. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  126. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  127. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  128. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  129. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  130. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  131. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  132. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  133. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  134. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  135. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  136. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  137. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  138. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  139. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  140. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  141. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  142. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  143. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  144. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  145. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  146. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  147. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  148. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  150. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  151. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  152. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  154. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  155. metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
  156. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  157. metaflow-stubs/plugins/perimeters.pyi +2 -2
  158. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  159. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  160. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  161. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  162. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  163. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  164. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  165. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  166. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  167. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  168. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  169. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  170. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  171. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  172. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  173. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  174. metaflow-stubs/profilers/__init__.pyi +2 -2
  175. metaflow-stubs/pylint_wrapper.pyi +2 -2
  176. metaflow-stubs/runner/__init__.pyi +2 -2
  177. metaflow-stubs/runner/deployer.pyi +31 -31
  178. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  179. metaflow-stubs/runner/metaflow_runner.pyi +5 -5
  180. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  181. metaflow-stubs/runner/nbrun.pyi +2 -2
  182. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  183. metaflow-stubs/runner/utils.pyi +3 -3
  184. metaflow-stubs/system/__init__.pyi +2 -2
  185. metaflow-stubs/system/system_logger.pyi +3 -3
  186. metaflow-stubs/system/system_monitor.pyi +2 -2
  187. metaflow-stubs/tagging_util.pyi +2 -2
  188. metaflow-stubs/tuple_util.pyi +2 -2
  189. metaflow-stubs/user_configs/__init__.pyi +2 -2
  190. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  191. metaflow-stubs/user_configs/config_options.pyi +4 -4
  192. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  193. {ob_metaflow_stubs-6.0.3.145.dist-info → ob_metaflow_stubs-6.0.3.146.dist-info}/METADATA +1 -1
  194. ob_metaflow_stubs-6.0.3.146.dist-info/RECORD +197 -0
  195. ob_metaflow_stubs-6.0.3.145.dist-info/RECORD +0 -197
  196. {ob_metaflow_stubs-6.0.3.145.dist-info → ob_metaflow_stubs-6.0.3.146.dist-info}/WHEEL +0 -0
  197. {ob_metaflow_stubs-6.0.3.145.dist-info → ob_metaflow_stubs-6.0.3.146.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.14.3.1+obcheckpoint(0.1.8);ob(v1) #
4
- # Generated on 2025-02-24T18:52:39.248529 #
3
+ # MF version: 2.15.0.1+obcheckpoint(0.1.8);ob(v1) #
4
+ # Generated on 2025-02-25T21:49:22.675350 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -35,9 +35,9 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
+ from . import tuple_util as tuple_util
38
39
  from . import cards as cards
39
40
  from . import events as events
40
- from . import tuple_util as tuple_util
41
41
  from . import runner as runner
42
42
  from . import plugins as plugins
43
43
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
@@ -147,82 +147,21 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
147
147
  ...
148
148
 
149
149
  @typing.overload
150
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
151
- """
152
- Internal decorator to support Fast bakery
153
- Parameters
154
- ----------
155
- """
156
- ...
157
-
158
- @typing.overload
159
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
160
- ...
161
-
162
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
163
- """
164
- Internal decorator to support Fast bakery
165
- Parameters
166
- ----------
167
- """
168
- ...
169
-
170
- @typing.overload
171
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
150
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
172
151
  """
173
- Specifies the Conda environment for the step.
174
-
175
- Information in this decorator will augment any
176
- attributes set in the `@conda_base` flow-level decorator. Hence,
177
- you can use `@conda_base` to set packages required by all
178
- steps and use `@conda` to specify step-specific overrides.
179
-
180
-
181
152
  Parameters
182
153
  ----------
183
- packages : Dict[str, str], default {}
184
- Packages to use for this step. The key is the name of the package
185
- and the value is the version to use.
186
- libraries : Dict[str, str], default {}
187
- Supported for backward compatibility. When used with packages, packages will take precedence.
188
- python : str, optional, default None
189
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
190
- that the version used will correspond to the version of the Python interpreter used to start the run.
191
- disabled : bool, default False
192
- If set to True, disables @conda.
193
154
  """
194
155
  ...
195
156
 
196
157
  @typing.overload
197
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
198
- ...
199
-
200
- @typing.overload
201
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
158
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
202
159
  ...
203
160
 
204
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
161
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
205
162
  """
206
- Specifies the Conda environment for the step.
207
-
208
- Information in this decorator will augment any
209
- attributes set in the `@conda_base` flow-level decorator. Hence,
210
- you can use `@conda_base` to set packages required by all
211
- steps and use `@conda` to specify step-specific overrides.
212
-
213
-
214
163
  Parameters
215
164
  ----------
216
- packages : Dict[str, str], default {}
217
- Packages to use for this step. The key is the name of the package
218
- and the value is the version to use.
219
- libraries : Dict[str, str], default {}
220
- Supported for backward compatibility. When used with packages, packages will take precedence.
221
- python : str, optional, default None
222
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
223
- that the version used will correspond to the version of the Python interpreter used to start the run.
224
- disabled : bool, default False
225
- If set to True, disables @conda.
226
165
  """
227
166
  ...
228
167
 
@@ -275,39 +214,6 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
275
214
  """
276
215
  ...
277
216
 
278
- @typing.overload
279
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
280
- """
281
- Specifies environment variables to be set prior to the execution of a step.
282
-
283
-
284
- Parameters
285
- ----------
286
- vars : Dict[str, str], default {}
287
- Dictionary of environment variables to set.
288
- """
289
- ...
290
-
291
- @typing.overload
292
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
293
- ...
294
-
295
- @typing.overload
296
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
297
- ...
298
-
299
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
300
- """
301
- Specifies environment variables to be set prior to the execution of a step.
302
-
303
-
304
- Parameters
305
- ----------
306
- vars : Dict[str, str], default {}
307
- Dictionary of environment variables to set.
308
- """
309
- ...
310
-
311
217
  @typing.overload
312
218
  def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
313
219
  """
@@ -388,78 +294,130 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
388
294
  ...
389
295
 
390
296
  @typing.overload
391
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
297
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
392
298
  """
393
- Enables checkpointing for a step.
299
+ Specifies the PyPI packages for the step.
394
300
 
301
+ Information in this decorator will augment any
302
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
303
+ you can use `@pypi_base` to set packages required by all
304
+ steps and use `@pypi` to specify step-specific overrides.
395
305
 
396
306
 
397
307
  Parameters
398
308
  ----------
399
- load_policy : str, default: "fresh"
400
- The policy for loading the checkpoint. The following policies are supported:
401
- - "eager": Loads the the latest available checkpoint within the namespace.
402
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
403
- will be loaded at the start of the task.
404
- - "none": Do not load any checkpoint
405
- - "fresh": Loads the lastest checkpoint created within the running Task.
406
- This mode helps loading checkpoints across various retry attempts of the same task.
407
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
408
- created within the task will be loaded when the task is retries execution on failure.
409
-
410
- temp_dir_root : str, default: None
411
- The root directory under which `current.checkpoint.directory` will be created.
309
+ packages : Dict[str, str], default: {}
310
+ Packages to use for this step. The key is the name of the package
311
+ and the value is the version to use.
312
+ python : str, optional, default: None
313
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
314
+ that the version used will correspond to the version of the Python interpreter used to start the run.
412
315
  """
413
316
  ...
414
317
 
415
318
  @typing.overload
416
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
319
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
417
320
  ...
418
321
 
419
322
  @typing.overload
420
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
323
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
421
324
  ...
422
325
 
423
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
326
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
424
327
  """
425
- Enables checkpointing for a step.
426
-
328
+ Specifies the PyPI packages for the step.
427
329
 
330
+ Information in this decorator will augment any
331
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
332
+ you can use `@pypi_base` to set packages required by all
333
+ steps and use `@pypi` to specify step-specific overrides.
428
334
 
429
- Parameters
430
- ----------
431
- load_policy : str, default: "fresh"
432
- The policy for loading the checkpoint. The following policies are supported:
433
- - "eager": Loads the the latest available checkpoint within the namespace.
434
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
435
- will be loaded at the start of the task.
436
- - "none": Do not load any checkpoint
437
- - "fresh": Loads the lastest checkpoint created within the running Task.
438
- This mode helps loading checkpoints across various retry attempts of the same task.
439
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
440
- created within the task will be loaded when the task is retries execution on failure.
441
335
 
442
- temp_dir_root : str, default: None
443
- The root directory under which `current.checkpoint.directory` will be created.
444
- """
445
- ...
446
-
447
- @typing.overload
448
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
449
- """
450
336
  Parameters
451
337
  ----------
338
+ packages : Dict[str, str], default: {}
339
+ Packages to use for this step. The key is the name of the package
340
+ and the value is the version to use.
341
+ python : str, optional, default: None
342
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
343
+ that the version used will correspond to the version of the Python interpreter used to start the run.
452
344
  """
453
345
  ...
454
346
 
455
- @typing.overload
456
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
457
- ...
458
-
459
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
347
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
460
348
  """
349
+ Specifies that this step should execute on Kubernetes.
350
+
351
+
461
352
  Parameters
462
353
  ----------
354
+ cpu : int, default 1
355
+ Number of CPUs required for this step. If `@resources` is
356
+ also present, the maximum value from all decorators is used.
357
+ memory : int, default 4096
358
+ Memory size (in MB) required for this step. If
359
+ `@resources` is also present, the maximum value from all decorators is
360
+ used.
361
+ disk : int, default 10240
362
+ Disk size (in MB) required for this step. If
363
+ `@resources` is also present, the maximum value from all decorators is
364
+ used.
365
+ image : str, optional, default None
366
+ Docker image to use when launching on Kubernetes. If not specified, and
367
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
368
+ not, a default Docker image mapping to the current version of Python is used.
369
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
370
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
371
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
372
+ Kubernetes service account to use when launching pod in Kubernetes.
373
+ secrets : List[str], optional, default None
374
+ Kubernetes secrets to use when launching pod in Kubernetes. These
375
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
376
+ in Metaflow configuration.
377
+ node_selector: Union[Dict[str,str], str], optional, default None
378
+ Kubernetes node selector(s) to apply to the pod running the task.
379
+ Can be passed in as a comma separated string of values e.g.
380
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
381
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
382
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
383
+ Kubernetes namespace to use when launching pod in Kubernetes.
384
+ gpu : int, optional, default None
385
+ Number of GPUs required for this step. A value of zero implies that
386
+ the scheduled node should not have GPUs.
387
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
388
+ The vendor of the GPUs to be used for this step.
389
+ tolerations : List[str], default []
390
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
391
+ Kubernetes tolerations to use when launching pod in Kubernetes.
392
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
393
+ Kubernetes labels to use when launching pod in Kubernetes.
394
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
395
+ Kubernetes annotations to use when launching pod in Kubernetes.
396
+ use_tmpfs : bool, default False
397
+ This enables an explicit tmpfs mount for this step.
398
+ tmpfs_tempdir : bool, default True
399
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
400
+ tmpfs_size : int, optional, default: None
401
+ The value for the size (in MiB) of the tmpfs mount for this step.
402
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
403
+ memory allocated for this step.
404
+ tmpfs_path : str, optional, default /metaflow_temp
405
+ Path to tmpfs mount for this step.
406
+ persistent_volume_claims : Dict[str, str], optional, default None
407
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
408
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
409
+ shared_memory: int, optional
410
+ Shared memory size (in MiB) required for this step
411
+ port: int, optional
412
+ Port number to specify in the Kubernetes job object
413
+ compute_pool : str, optional, default None
414
+ Compute pool to be used for for this step.
415
+ If not specified, any accessible compute pool within the perimeter is used.
416
+ hostname_resolution_timeout: int, default 10 * 60
417
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
418
+ Only applicable when @parallel is used.
419
+ qos: str, default: Burstable
420
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
463
421
  """
464
422
  ...
465
423
 
@@ -489,143 +447,78 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
489
447
  ...
490
448
 
491
449
  @typing.overload
492
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
450
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
493
451
  """
494
- Enables loading / saving of models within a step.
495
-
452
+ Specifies the number of times the task corresponding
453
+ to a step needs to be retried.
496
454
 
455
+ This decorator is useful for handling transient errors, such as networking issues.
456
+ If your task contains operations that can't be retried safely, e.g. database updates,
457
+ it is advisable to annotate it with `@retry(times=0)`.
497
458
 
498
- Parameters
499
- ----------
500
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
501
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
502
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
503
- - `current.checkpoint`
504
- - `current.model`
505
- - `current.huggingface_hub`
459
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
460
+ decorator will execute a no-op task after all retries have been exhausted,
461
+ ensuring that the flow execution can continue.
506
462
 
507
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
508
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
509
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
510
463
 
511
- temp_dir_root : str, default: None
512
- The root directory under which `current.model.loaded` will store loaded models
464
+ Parameters
465
+ ----------
466
+ times : int, default 3
467
+ Number of times to retry this task.
468
+ minutes_between_retries : int, default 2
469
+ Number of minutes between retries.
513
470
  """
514
471
  ...
515
472
 
516
473
  @typing.overload
517
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
474
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
518
475
  ...
519
476
 
520
477
  @typing.overload
521
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
522
- ...
523
-
524
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
525
- """
526
- Enables loading / saving of models within a step.
527
-
528
-
529
-
530
- Parameters
531
- ----------
532
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
533
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
534
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
535
- - `current.checkpoint`
536
- - `current.model`
537
- - `current.huggingface_hub`
538
-
539
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
540
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
541
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
542
-
543
- temp_dir_root : str, default: None
544
- The root directory under which `current.model.loaded` will store loaded models
545
- """
478
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
546
479
  ...
547
480
 
548
- def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
481
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
549
482
  """
550
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
551
-
552
- User code call
553
- -----------
554
- @nim(
555
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
556
- backend='managed'
557
- )
483
+ Specifies the number of times the task corresponding
484
+ to a step needs to be retried.
558
485
 
559
- Valid backend options
560
- ---------------------
561
- - 'managed': Outerbounds selects a compute provider based on the model.
486
+ This decorator is useful for handling transient errors, such as networking issues.
487
+ If your task contains operations that can't be retried safely, e.g. database updates,
488
+ it is advisable to annotate it with `@retry(times=0)`.
562
489
 
563
- Valid model options
564
- ----------------
565
- - 'meta/llama3-8b-instruct': 8B parameter model
566
- - 'meta/llama3-70b-instruct': 70B parameter model
567
- - any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
490
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
491
+ decorator will execute a no-op task after all retries have been exhausted,
492
+ ensuring that the flow execution can continue.
568
493
 
569
494
 
570
495
  Parameters
571
496
  ----------
572
- models: list[NIM]
573
- List of NIM containers running models in sidecars.
574
- backend: str
575
- Compute provider to run the NIM container.
576
- queue_timeout : int
577
- Time to keep the job in NVCF's queue.
497
+ times : int, default 3
498
+ Number of times to retry this task.
499
+ minutes_between_retries : int, default 2
500
+ Number of minutes between retries.
578
501
  """
579
502
  ...
580
503
 
581
504
  @typing.overload
582
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
505
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
583
506
  """
584
- Specifies that the step will success under all circumstances.
585
-
586
- The decorator will create an optional artifact, specified by `var`, which
587
- contains the exception raised. You can use it to detect the presence
588
- of errors, indicating that all happy-path artifacts produced by the step
589
- are missing.
590
-
591
-
507
+ Internal decorator to support Fast bakery
592
508
  Parameters
593
509
  ----------
594
- var : str, optional, default None
595
- Name of the artifact in which to store the caught exception.
596
- If not specified, the exception is not stored.
597
- print_exception : bool, default True
598
- Determines whether or not the exception is printed to
599
- stdout when caught.
600
510
  """
601
511
  ...
602
512
 
603
513
  @typing.overload
604
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
605
- ...
606
-
607
- @typing.overload
608
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
514
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
609
515
  ...
610
516
 
611
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
517
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
612
518
  """
613
- Specifies that the step will success under all circumstances.
614
-
615
- The decorator will create an optional artifact, specified by `var`, which
616
- contains the exception raised. You can use it to detect the presence
617
- of errors, indicating that all happy-path artifacts produced by the step
618
- are missing.
619
-
620
-
519
+ Internal decorator to support Fast bakery
621
520
  Parameters
622
521
  ----------
623
- var : str, optional, default None
624
- Name of the artifact in which to store the caught exception.
625
- If not specified, the exception is not stored.
626
- print_exception : bool, default True
627
- Determines whether or not the exception is printed to
628
- stdout when caught.
629
522
  """
630
523
  ...
631
524
 
@@ -688,80 +581,201 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
688
581
  """
689
582
  ...
690
583
 
691
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
584
+ @typing.overload
585
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
692
586
  """
693
- Specifies that this step should execute on Kubernetes.
587
+ Specifies the Conda environment for the step.
588
+
589
+ Information in this decorator will augment any
590
+ attributes set in the `@conda_base` flow-level decorator. Hence,
591
+ you can use `@conda_base` to set packages required by all
592
+ steps and use `@conda` to specify step-specific overrides.
694
593
 
695
594
 
696
595
  Parameters
697
596
  ----------
698
- cpu : int, default 1
699
- Number of CPUs required for this step. If `@resources` is
700
- also present, the maximum value from all decorators is used.
701
- memory : int, default 4096
702
- Memory size (in MB) required for this step. If
703
- `@resources` is also present, the maximum value from all decorators is
704
- used.
705
- disk : int, default 10240
706
- Disk size (in MB) required for this step. If
707
- `@resources` is also present, the maximum value from all decorators is
708
- used.
709
- image : str, optional, default None
710
- Docker image to use when launching on Kubernetes. If not specified, and
711
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
712
- not, a default Docker image mapping to the current version of Python is used.
713
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
714
- If given, the imagePullPolicy to be applied to the Docker image of the step.
715
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
716
- Kubernetes service account to use when launching pod in Kubernetes.
717
- secrets : List[str], optional, default None
718
- Kubernetes secrets to use when launching pod in Kubernetes. These
719
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
720
- in Metaflow configuration.
721
- node_selector: Union[Dict[str,str], str], optional, default None
722
- Kubernetes node selector(s) to apply to the pod running the task.
723
- Can be passed in as a comma separated string of values e.g.
724
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
725
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
726
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
727
- Kubernetes namespace to use when launching pod in Kubernetes.
728
- gpu : int, optional, default None
729
- Number of GPUs required for this step. A value of zero implies that
730
- the scheduled node should not have GPUs.
731
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
732
- The vendor of the GPUs to be used for this step.
733
- tolerations : List[str], default []
734
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
735
- Kubernetes tolerations to use when launching pod in Kubernetes.
736
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
737
- Kubernetes labels to use when launching pod in Kubernetes.
738
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
739
- Kubernetes annotations to use when launching pod in Kubernetes.
740
- use_tmpfs : bool, default False
741
- This enables an explicit tmpfs mount for this step.
742
- tmpfs_tempdir : bool, default True
743
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
744
- tmpfs_size : int, optional, default: None
745
- The value for the size (in MiB) of the tmpfs mount for this step.
746
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
747
- memory allocated for this step.
748
- tmpfs_path : str, optional, default /metaflow_temp
749
- Path to tmpfs mount for this step.
750
- persistent_volume_claims : Dict[str, str], optional, default None
751
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
752
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
753
- shared_memory: int, optional
754
- Shared memory size (in MiB) required for this step
755
- port: int, optional
756
- Port number to specify in the Kubernetes job object
757
- compute_pool : str, optional, default None
758
- Compute pool to be used for for this step.
759
- If not specified, any accessible compute pool within the perimeter is used.
760
- hostname_resolution_timeout: int, default 10 * 60
761
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
762
- Only applicable when @parallel is used.
763
- qos: str, default: Burstable
764
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
597
+ packages : Dict[str, str], default {}
598
+ Packages to use for this step. The key is the name of the package
599
+ and the value is the version to use.
600
+ libraries : Dict[str, str], default {}
601
+ Supported for backward compatibility. When used with packages, packages will take precedence.
602
+ python : str, optional, default None
603
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
604
+ that the version used will correspond to the version of the Python interpreter used to start the run.
605
+ disabled : bool, default False
606
+ If set to True, disables @conda.
607
+ """
608
+ ...
609
+
610
+ @typing.overload
611
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
612
+ ...
613
+
614
+ @typing.overload
615
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
616
+ ...
617
+
618
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
619
+ """
620
+ Specifies the Conda environment for the step.
621
+
622
+ Information in this decorator will augment any
623
+ attributes set in the `@conda_base` flow-level decorator. Hence,
624
+ you can use `@conda_base` to set packages required by all
625
+ steps and use `@conda` to specify step-specific overrides.
626
+
627
+
628
+ Parameters
629
+ ----------
630
+ packages : Dict[str, str], default {}
631
+ Packages to use for this step. The key is the name of the package
632
+ and the value is the version to use.
633
+ libraries : Dict[str, str], default {}
634
+ Supported for backward compatibility. When used with packages, packages will take precedence.
635
+ python : str, optional, default None
636
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
637
+ that the version used will correspond to the version of the Python interpreter used to start the run.
638
+ disabled : bool, default False
639
+ If set to True, disables @conda.
640
+ """
641
+ ...
642
+
643
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
644
+ """
645
+ Specifies that this step should execute on DGX cloud.
646
+
647
+
648
+ Parameters
649
+ ----------
650
+ gpu : int
651
+ Number of GPUs to use.
652
+ gpu_type : str
653
+ Type of Nvidia GPU to use.
654
+ queue_timeout : int
655
+ Time to keep the job in NVCF's queue.
656
+ """
657
+ ...
658
+
659
+ def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
660
+ """
661
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
662
+
663
+ User code call
664
+ -----------
665
+ @nim(
666
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
667
+ backend='managed'
668
+ )
669
+
670
+ Valid backend options
671
+ ---------------------
672
+ - 'managed': Outerbounds selects a compute provider based on the model.
673
+
674
+ Valid model options
675
+ ----------------
676
+ - 'meta/llama3-8b-instruct': 8B parameter model
677
+ - 'meta/llama3-70b-instruct': 70B parameter model
678
+ - any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
679
+
680
+
681
+ Parameters
682
+ ----------
683
+ models: list[NIM]
684
+ List of NIM containers running models in sidecars.
685
+ backend: str
686
+ Compute provider to run the NIM container.
687
+ queue_timeout : int
688
+ Time to keep the job in NVCF's queue.
689
+ """
690
+ ...
691
+
692
+ @typing.overload
693
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
694
+ """
695
+ Specifies environment variables to be set prior to the execution of a step.
696
+
697
+
698
+ Parameters
699
+ ----------
700
+ vars : Dict[str, str], default {}
701
+ Dictionary of environment variables to set.
702
+ """
703
+ ...
704
+
705
+ @typing.overload
706
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
707
+ ...
708
+
709
+ @typing.overload
710
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
711
+ ...
712
+
713
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
714
+ """
715
+ Specifies environment variables to be set prior to the execution of a step.
716
+
717
+
718
+ Parameters
719
+ ----------
720
+ vars : Dict[str, str], default {}
721
+ Dictionary of environment variables to set.
722
+ """
723
+ ...
724
+
725
+ @typing.overload
726
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
727
+ """
728
+ Enables checkpointing for a step.
729
+
730
+
731
+
732
+ Parameters
733
+ ----------
734
+ load_policy : str, default: "fresh"
735
+ The policy for loading the checkpoint. The following policies are supported:
736
+ - "eager": Loads the the latest available checkpoint within the namespace.
737
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
738
+ will be loaded at the start of the task.
739
+ - "none": Do not load any checkpoint
740
+ - "fresh": Loads the lastest checkpoint created within the running Task.
741
+ This mode helps loading checkpoints across various retry attempts of the same task.
742
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
743
+ created within the task will be loaded when the task is retries execution on failure.
744
+
745
+ temp_dir_root : str, default: None
746
+ The root directory under which `current.checkpoint.directory` will be created.
747
+ """
748
+ ...
749
+
750
+ @typing.overload
751
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
752
+ ...
753
+
754
+ @typing.overload
755
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
756
+ ...
757
+
758
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
759
+ """
760
+ Enables checkpointing for a step.
761
+
762
+
763
+
764
+ Parameters
765
+ ----------
766
+ load_policy : str, default: "fresh"
767
+ The policy for loading the checkpoint. The following policies are supported:
768
+ - "eager": Loads the the latest available checkpoint within the namespace.
769
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
770
+ will be loaded at the start of the task.
771
+ - "none": Do not load any checkpoint
772
+ - "fresh": Loads the lastest checkpoint created within the running Task.
773
+ This mode helps loading checkpoints across various retry attempts of the same task.
774
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
775
+ created within the task will be loaded when the task is retries execution on failure.
776
+
777
+ temp_dir_root : str, default: None
778
+ The root directory under which `current.checkpoint.directory` will be created.
765
779
  """
766
780
  ...
767
781
 
@@ -798,22 +812,6 @@ def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.U
798
812
  """
799
813
  ...
800
814
 
801
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
802
- """
803
- Specifies that this step should execute on DGX cloud.
804
-
805
-
806
- Parameters
807
- ----------
808
- gpu : int
809
- Number of GPUs to use.
810
- gpu_type : str
811
- Type of Nvidia GPU to use.
812
- queue_timeout : int
813
- Time to keep the job in NVCF's queue.
814
- """
815
- ...
816
-
817
815
  @typing.overload
818
816
  def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
819
817
  """
@@ -850,243 +848,196 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
850
848
  ...
851
849
 
852
850
  @typing.overload
853
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
851
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
854
852
  """
855
- Specifies the number of times the task corresponding
856
- to a step needs to be retried.
857
-
858
- This decorator is useful for handling transient errors, such as networking issues.
859
- If your task contains operations that can't be retried safely, e.g. database updates,
860
- it is advisable to annotate it with `@retry(times=0)`.
853
+ Enables loading / saving of models within a step.
861
854
 
862
- This can be used in conjunction with the `@catch` decorator. The `@catch`
863
- decorator will execute a no-op task after all retries have been exhausted,
864
- ensuring that the flow execution can continue.
865
855
 
866
856
 
867
857
  Parameters
868
858
  ----------
869
- times : int, default 3
870
- Number of times to retry this task.
871
- minutes_between_retries : int, default 2
872
- Number of minutes between retries.
859
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
860
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
861
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
862
+ - `current.checkpoint`
863
+ - `current.model`
864
+ - `current.huggingface_hub`
865
+
866
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
867
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
868
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
869
+
870
+ temp_dir_root : str, default: None
871
+ The root directory under which `current.model.loaded` will store loaded models
873
872
  """
874
873
  ...
875
874
 
876
875
  @typing.overload
877
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
876
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
878
877
  ...
879
878
 
880
879
  @typing.overload
881
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
880
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
882
881
  ...
883
882
 
884
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
883
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
885
884
  """
886
- Specifies the number of times the task corresponding
887
- to a step needs to be retried.
888
-
889
- This decorator is useful for handling transient errors, such as networking issues.
890
- If your task contains operations that can't be retried safely, e.g. database updates,
891
- it is advisable to annotate it with `@retry(times=0)`.
885
+ Enables loading / saving of models within a step.
892
886
 
893
- This can be used in conjunction with the `@catch` decorator. The `@catch`
894
- decorator will execute a no-op task after all retries have been exhausted,
895
- ensuring that the flow execution can continue.
896
887
 
897
888
 
898
889
  Parameters
899
890
  ----------
900
- times : int, default 3
901
- Number of times to retry this task.
902
- minutes_between_retries : int, default 2
903
- Number of minutes between retries.
904
- """
905
- ...
906
-
907
- @typing.overload
908
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
909
- """
910
- Specifies the PyPI packages for the step.
911
-
912
- Information in this decorator will augment any
913
- attributes set in the `@pyi_base` flow-level decorator. Hence,
914
- you can use `@pypi_base` to set packages required by all
915
- steps and use `@pypi` to specify step-specific overrides.
891
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
892
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
893
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
894
+ - `current.checkpoint`
895
+ - `current.model`
896
+ - `current.huggingface_hub`
916
897
 
898
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
899
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
900
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
917
901
 
918
- Parameters
919
- ----------
920
- packages : Dict[str, str], default: {}
921
- Packages to use for this step. The key is the name of the package
922
- and the value is the version to use.
923
- python : str, optional, default: None
924
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
925
- that the version used will correspond to the version of the Python interpreter used to start the run.
902
+ temp_dir_root : str, default: None
903
+ The root directory under which `current.model.loaded` will store loaded models
926
904
  """
927
905
  ...
928
906
 
929
907
  @typing.overload
930
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
931
- ...
932
-
933
- @typing.overload
934
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
935
- ...
936
-
937
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
908
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
938
909
  """
939
- Specifies the PyPI packages for the step.
910
+ Specifies that the step will success under all circumstances.
940
911
 
941
- Information in this decorator will augment any
942
- attributes set in the `@pyi_base` flow-level decorator. Hence,
943
- you can use `@pypi_base` to set packages required by all
944
- steps and use `@pypi` to specify step-specific overrides.
912
+ The decorator will create an optional artifact, specified by `var`, which
913
+ contains the exception raised. You can use it to detect the presence
914
+ of errors, indicating that all happy-path artifacts produced by the step
915
+ are missing.
945
916
 
946
917
 
947
918
  Parameters
948
919
  ----------
949
- packages : Dict[str, str], default: {}
950
- Packages to use for this step. The key is the name of the package
951
- and the value is the version to use.
952
- python : str, optional, default: None
953
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
954
- that the version used will correspond to the version of the Python interpreter used to start the run.
920
+ var : str, optional, default None
921
+ Name of the artifact in which to store the caught exception.
922
+ If not specified, the exception is not stored.
923
+ print_exception : bool, default True
924
+ Determines whether or not the exception is printed to
925
+ stdout when caught.
955
926
  """
956
927
  ...
957
928
 
958
929
  @typing.overload
959
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
960
- """
961
- Specifies the PyPI packages for all steps of the flow.
962
-
963
- Use `@pypi_base` to set common packages required by all
964
- steps and use `@pypi` to specify step-specific overrides.
965
-
966
- Parameters
967
- ----------
968
- packages : Dict[str, str], default: {}
969
- Packages to use for this flow. The key is the name of the package
970
- and the value is the version to use.
971
- python : str, optional, default: None
972
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
973
- that the version used will correspond to the version of the Python interpreter used to start the run.
974
- """
930
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
975
931
  ...
976
932
 
977
933
  @typing.overload
978
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
934
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
979
935
  ...
980
936
 
981
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
937
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
982
938
  """
983
- Specifies the PyPI packages for all steps of the flow.
939
+ Specifies that the step will success under all circumstances.
940
+
941
+ The decorator will create an optional artifact, specified by `var`, which
942
+ contains the exception raised. You can use it to detect the presence
943
+ of errors, indicating that all happy-path artifacts produced by the step
944
+ are missing.
984
945
 
985
- Use `@pypi_base` to set common packages required by all
986
- steps and use `@pypi` to specify step-specific overrides.
987
946
 
988
947
  Parameters
989
948
  ----------
990
- packages : Dict[str, str], default: {}
991
- Packages to use for this flow. The key is the name of the package
992
- and the value is the version to use.
993
- python : str, optional, default: None
994
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
995
- that the version used will correspond to the version of the Python interpreter used to start the run.
949
+ var : str, optional, default None
950
+ Name of the artifact in which to store the caught exception.
951
+ If not specified, the exception is not stored.
952
+ print_exception : bool, default True
953
+ Determines whether or not the exception is printed to
954
+ stdout when caught.
996
955
  """
997
956
  ...
998
957
 
999
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1000
- """
1001
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1002
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1003
-
1004
-
1005
- Parameters
1006
- ----------
1007
- timeout : int
1008
- Time, in seconds before the task times out and fails. (Default: 3600)
1009
- poke_interval : int
1010
- Time in seconds that the job should wait in between each try. (Default: 60)
1011
- mode : str
1012
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1013
- exponential_backoff : bool
1014
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1015
- pool : str
1016
- the slot pool this task should run in,
1017
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1018
- soft_fail : bool
1019
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1020
- name : str
1021
- Name of the sensor on Airflow
1022
- description : str
1023
- Description of sensor in the Airflow UI
1024
- external_dag_id : str
1025
- The dag_id that contains the task you want to wait for.
1026
- external_task_ids : List[str]
1027
- The list of task_ids that you want to wait for.
1028
- If None (default value) the sensor waits for the DAG. (Default: None)
1029
- allowed_states : List[str]
1030
- Iterable of allowed states, (Default: ['success'])
1031
- failed_states : List[str]
1032
- Iterable of failed or dis-allowed states. (Default: None)
1033
- execution_delta : datetime.timedelta
1034
- time difference with the previous execution to look at,
1035
- the default is the same logical date as the current task or DAG. (Default: None)
1036
- check_existence: bool
1037
- Set to True to check if the external task exists or check if
1038
- the DAG to wait for exists. (Default: True)
958
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
959
+ """
960
+ Specifies what flows belong to the same project.
961
+
962
+ A project-specific namespace is created for all flows that
963
+ use the same `@project(name)`.
964
+
965
+
966
+ Parameters
967
+ ----------
968
+ name : str
969
+ Project name. Make sure that the name is unique amongst all
970
+ projects that use the same production scheduler. The name may
971
+ contain only lowercase alphanumeric characters and underscores.
972
+
973
+ branch : Optional[str], default None
974
+ The branch to use. If not specified, the branch is set to
975
+ `user.<username>` unless `production` is set to `True`. This can
976
+ also be set on the command line using `--branch` as a top-level option.
977
+ It is an error to specify `branch` in the decorator and on the command line.
978
+
979
+ production : bool, default False
980
+ Whether or not the branch is the production branch. This can also be set on the
981
+ command line using `--production` as a top-level option. It is an error to specify
982
+ `production` in the decorator and on the command line.
983
+ The project branch name will be:
984
+ - if `branch` is specified:
985
+ - if `production` is True: `prod.<branch>`
986
+ - if `production` is False: `test.<branch>`
987
+ - if `branch` is not specified:
988
+ - if `production` is True: `prod`
989
+ - if `production` is False: `user.<username>`
1039
990
  """
1040
991
  ...
1041
992
 
1042
993
  @typing.overload
1043
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
994
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1044
995
  """
1045
- Specifies the Conda environment for all steps of the flow.
1046
-
1047
- Use `@conda_base` to set common libraries required by all
1048
- steps and use `@conda` to specify step-specific additions.
996
+ Specifies the times when the flow should be run when running on a
997
+ production scheduler.
1049
998
 
1050
999
 
1051
1000
  Parameters
1052
1001
  ----------
1053
- packages : Dict[str, str], default {}
1054
- Packages to use for this flow. The key is the name of the package
1055
- and the value is the version to use.
1056
- libraries : Dict[str, str], default {}
1057
- Supported for backward compatibility. When used with packages, packages will take precedence.
1058
- python : str, optional, default None
1059
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1060
- that the version used will correspond to the version of the Python interpreter used to start the run.
1061
- disabled : bool, default False
1062
- If set to True, disables Conda.
1002
+ hourly : bool, default False
1003
+ Run the workflow hourly.
1004
+ daily : bool, default True
1005
+ Run the workflow daily.
1006
+ weekly : bool, default False
1007
+ Run the workflow weekly.
1008
+ cron : str, optional, default None
1009
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1010
+ specified by this expression.
1011
+ timezone : str, optional, default None
1012
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1013
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1063
1014
  """
1064
1015
  ...
1065
1016
 
1066
1017
  @typing.overload
1067
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1018
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1068
1019
  ...
1069
1020
 
1070
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1021
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1071
1022
  """
1072
- Specifies the Conda environment for all steps of the flow.
1073
-
1074
- Use `@conda_base` to set common libraries required by all
1075
- steps and use `@conda` to specify step-specific additions.
1023
+ Specifies the times when the flow should be run when running on a
1024
+ production scheduler.
1076
1025
 
1077
1026
 
1078
1027
  Parameters
1079
1028
  ----------
1080
- packages : Dict[str, str], default {}
1081
- Packages to use for this flow. The key is the name of the package
1082
- and the value is the version to use.
1083
- libraries : Dict[str, str], default {}
1084
- Supported for backward compatibility. When used with packages, packages will take precedence.
1085
- python : str, optional, default None
1086
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1087
- that the version used will correspond to the version of the Python interpreter used to start the run.
1088
- disabled : bool, default False
1089
- If set to True, disables Conda.
1029
+ hourly : bool, default False
1030
+ Run the workflow hourly.
1031
+ daily : bool, default True
1032
+ Run the workflow daily.
1033
+ weekly : bool, default False
1034
+ Run the workflow weekly.
1035
+ cron : str, optional, default None
1036
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1037
+ specified by this expression.
1038
+ timezone : str, optional, default None
1039
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1040
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1090
1041
  """
1091
1042
  ...
1092
1043
 
@@ -1183,38 +1134,46 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1183
1134
  """
1184
1135
  ...
1185
1136
 
1186
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1137
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1187
1138
  """
1188
- Specifies what flows belong to the same project.
1189
-
1190
- A project-specific namespace is created for all flows that
1191
- use the same `@project(name)`.
1139
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1140
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1192
1141
 
1193
1142
 
1194
1143
  Parameters
1195
1144
  ----------
1145
+ timeout : int
1146
+ Time, in seconds before the task times out and fails. (Default: 3600)
1147
+ poke_interval : int
1148
+ Time in seconds that the job should wait in between each try. (Default: 60)
1149
+ mode : str
1150
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1151
+ exponential_backoff : bool
1152
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1153
+ pool : str
1154
+ the slot pool this task should run in,
1155
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1156
+ soft_fail : bool
1157
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1196
1158
  name : str
1197
- Project name. Make sure that the name is unique amongst all
1198
- projects that use the same production scheduler. The name may
1199
- contain only lowercase alphanumeric characters and underscores.
1200
-
1201
- branch : Optional[str], default None
1202
- The branch to use. If not specified, the branch is set to
1203
- `user.<username>` unless `production` is set to `True`. This can
1204
- also be set on the command line using `--branch` as a top-level option.
1205
- It is an error to specify `branch` in the decorator and on the command line.
1206
-
1207
- production : bool, default False
1208
- Whether or not the branch is the production branch. This can also be set on the
1209
- command line using `--production` as a top-level option. It is an error to specify
1210
- `production` in the decorator and on the command line.
1211
- The project branch name will be:
1212
- - if `branch` is specified:
1213
- - if `production` is True: `prod.<branch>`
1214
- - if `production` is False: `test.<branch>`
1215
- - if `branch` is not specified:
1216
- - if `production` is True: `prod`
1217
- - if `production` is False: `user.<username>`
1159
+ Name of the sensor on Airflow
1160
+ description : str
1161
+ Description of sensor in the Airflow UI
1162
+ external_dag_id : str
1163
+ The dag_id that contains the task you want to wait for.
1164
+ external_task_ids : List[str]
1165
+ The list of task_ids that you want to wait for.
1166
+ If None (default value) the sensor waits for the DAG. (Default: None)
1167
+ allowed_states : List[str]
1168
+ Iterable of allowed states, (Default: ['success'])
1169
+ failed_states : List[str]
1170
+ Iterable of failed or dis-allowed states. (Default: None)
1171
+ execution_delta : datetime.timedelta
1172
+ time difference with the previous execution to look at,
1173
+ the default is the same logical date as the current task or DAG. (Default: None)
1174
+ check_existence: bool
1175
+ Set to True to check if the external task exists or check if
1176
+ the DAG to wait for exists. (Default: True)
1218
1177
  """
1219
1178
  ...
1220
1179
 
@@ -1319,6 +1278,57 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1319
1278
  """
1320
1279
  ...
1321
1280
 
1281
+ @typing.overload
1282
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1283
+ """
1284
+ Specifies the Conda environment for all steps of the flow.
1285
+
1286
+ Use `@conda_base` to set common libraries required by all
1287
+ steps and use `@conda` to specify step-specific additions.
1288
+
1289
+
1290
+ Parameters
1291
+ ----------
1292
+ packages : Dict[str, str], default {}
1293
+ Packages to use for this flow. The key is the name of the package
1294
+ and the value is the version to use.
1295
+ libraries : Dict[str, str], default {}
1296
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1297
+ python : str, optional, default None
1298
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1299
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1300
+ disabled : bool, default False
1301
+ If set to True, disables Conda.
1302
+ """
1303
+ ...
1304
+
1305
+ @typing.overload
1306
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1307
+ ...
1308
+
1309
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1310
+ """
1311
+ Specifies the Conda environment for all steps of the flow.
1312
+
1313
+ Use `@conda_base` to set common libraries required by all
1314
+ steps and use `@conda` to specify step-specific additions.
1315
+
1316
+
1317
+ Parameters
1318
+ ----------
1319
+ packages : Dict[str, str], default {}
1320
+ Packages to use for this flow. The key is the name of the package
1321
+ and the value is the version to use.
1322
+ libraries : Dict[str, str], default {}
1323
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1324
+ python : str, optional, default None
1325
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1326
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1327
+ disabled : bool, default False
1328
+ If set to True, disables Conda.
1329
+ """
1330
+ ...
1331
+
1322
1332
  def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1323
1333
  """
1324
1334
  The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
@@ -1363,53 +1373,43 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1363
1373
  ...
1364
1374
 
1365
1375
  @typing.overload
1366
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1376
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1367
1377
  """
1368
- Specifies the times when the flow should be run when running on a
1369
- production scheduler.
1378
+ Specifies the PyPI packages for all steps of the flow.
1370
1379
 
1380
+ Use `@pypi_base` to set common packages required by all
1381
+ steps and use `@pypi` to specify step-specific overrides.
1371
1382
 
1372
1383
  Parameters
1373
1384
  ----------
1374
- hourly : bool, default False
1375
- Run the workflow hourly.
1376
- daily : bool, default True
1377
- Run the workflow daily.
1378
- weekly : bool, default False
1379
- Run the workflow weekly.
1380
- cron : str, optional, default None
1381
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1382
- specified by this expression.
1383
- timezone : str, optional, default None
1384
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1385
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1385
+ packages : Dict[str, str], default: {}
1386
+ Packages to use for this flow. The key is the name of the package
1387
+ and the value is the version to use.
1388
+ python : str, optional, default: None
1389
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1390
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1386
1391
  """
1387
1392
  ...
1388
1393
 
1389
1394
  @typing.overload
1390
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1395
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1391
1396
  ...
1392
1397
 
1393
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1398
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1394
1399
  """
1395
- Specifies the times when the flow should be run when running on a
1396
- production scheduler.
1400
+ Specifies the PyPI packages for all steps of the flow.
1397
1401
 
1402
+ Use `@pypi_base` to set common packages required by all
1403
+ steps and use `@pypi` to specify step-specific overrides.
1398
1404
 
1399
1405
  Parameters
1400
1406
  ----------
1401
- hourly : bool, default False
1402
- Run the workflow hourly.
1403
- daily : bool, default True
1404
- Run the workflow daily.
1405
- weekly : bool, default False
1406
- Run the workflow weekly.
1407
- cron : str, optional, default None
1408
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1409
- specified by this expression.
1410
- timezone : str, optional, default None
1411
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1412
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1407
+ packages : Dict[str, str], default: {}
1408
+ Packages to use for this flow. The key is the name of the package
1409
+ and the value is the version to use.
1410
+ python : str, optional, default: None
1411
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1412
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1413
1413
  """
1414
1414
  ...
1415
1415