ob-metaflow-stubs 6.0.3.168rc0__py2.py3-none-any.whl → 6.0.3.169__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. metaflow-stubs/__init__.pyi +562 -595
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +107 -107
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +4 -4
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +4 -4
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  80. metaflow-stubs/multicore_utils.pyi +2 -2
  81. metaflow-stubs/parameters.pyi +3 -3
  82. metaflow-stubs/plugins/__init__.pyi +10 -10
  83. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  84. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  85. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  86. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  88. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  89. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  90. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  92. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  93. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  94. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  95. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  96. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -4
  97. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  99. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  100. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  102. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  103. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  105. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  106. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  108. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  109. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  110. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  111. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  112. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  113. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  114. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  115. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  116. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  117. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  118. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  119. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  120. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  121. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  123. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  124. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  126. metaflow-stubs/plugins/cards/card_modules/basic.pyi +4 -4
  127. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  128. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  129. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  130. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  131. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  132. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  133. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  134. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  135. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  136. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  138. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  139. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  140. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  141. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  142. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  143. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  144. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  145. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  146. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  147. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  148. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  150. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  151. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  152. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  153. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  154. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  155. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  156. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  157. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  158. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  159. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  160. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  161. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  162. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  163. metaflow-stubs/plugins/perimeters.pyi +2 -2
  164. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  165. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  166. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  167. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  168. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  169. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  170. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  171. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  172. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  173. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  174. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  175. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  176. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  177. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  178. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  179. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  180. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  181. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  182. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  183. metaflow-stubs/profilers/__init__.pyi +2 -2
  184. metaflow-stubs/pylint_wrapper.pyi +2 -2
  185. metaflow-stubs/runner/__init__.pyi +2 -2
  186. metaflow-stubs/runner/deployer.pyi +6 -6
  187. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  188. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  189. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  190. metaflow-stubs/runner/nbrun.pyi +2 -2
  191. metaflow-stubs/runner/subprocess_manager.pyi +3 -3
  192. metaflow-stubs/runner/utils.pyi +3 -3
  193. metaflow-stubs/system/__init__.pyi +2 -2
  194. metaflow-stubs/system/system_logger.pyi +3 -3
  195. metaflow-stubs/system/system_monitor.pyi +2 -2
  196. metaflow-stubs/tagging_util.pyi +2 -2
  197. metaflow-stubs/tuple_util.pyi +2 -2
  198. metaflow-stubs/user_configs/__init__.pyi +2 -2
  199. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  200. metaflow-stubs/user_configs/config_options.pyi +3 -3
  201. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  202. {ob_metaflow_stubs-6.0.3.168rc0.dist-info → ob_metaflow_stubs-6.0.3.169.dist-info}/METADATA +1 -1
  203. ob_metaflow_stubs-6.0.3.169.dist-info/RECORD +206 -0
  204. ob_metaflow_stubs-6.0.3.168rc0.dist-info/RECORD +0 -206
  205. {ob_metaflow_stubs-6.0.3.168rc0.dist-info → ob_metaflow_stubs-6.0.3.169.dist-info}/WHEEL +0 -0
  206. {ob_metaflow_stubs-6.0.3.168rc0.dist-info → ob_metaflow_stubs-6.0.3.169.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.11.2+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-05-19T23:34:02.941331 #
3
+ # MF version: 2.15.14.1+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-05-21T14:02:14.742508 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -154,208 +154,195 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
154
154
  ...
155
155
 
156
156
  @typing.overload
157
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
157
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
158
158
  """
159
- Enables checkpointing for a step.
159
+ Specifies that the step will success under all circumstances.
160
160
 
161
+ The decorator will create an optional artifact, specified by `var`, which
162
+ contains the exception raised. You can use it to detect the presence
163
+ of errors, indicating that all happy-path artifacts produced by the step
164
+ are missing.
161
165
 
162
166
 
163
167
  Parameters
164
168
  ----------
165
- load_policy : str, default: "fresh"
166
- The policy for loading the checkpoint. The following policies are supported:
167
- - "eager": Loads the the latest available checkpoint within the namespace.
168
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
169
- will be loaded at the start of the task.
170
- - "none": Do not load any checkpoint
171
- - "fresh": Loads the lastest checkpoint created within the running Task.
172
- This mode helps loading checkpoints across various retry attempts of the same task.
173
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
174
- created within the task will be loaded when the task is retries execution on failure.
175
-
176
- temp_dir_root : str, default: None
177
- The root directory under which `current.checkpoint.directory` will be created.
169
+ var : str, optional, default None
170
+ Name of the artifact in which to store the caught exception.
171
+ If not specified, the exception is not stored.
172
+ print_exception : bool, default True
173
+ Determines whether or not the exception is printed to
174
+ stdout when caught.
178
175
  """
179
176
  ...
180
177
 
181
178
  @typing.overload
182
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
179
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
183
180
  ...
184
181
 
185
182
  @typing.overload
186
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
183
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
187
184
  ...
188
185
 
189
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
186
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
190
187
  """
191
- Enables checkpointing for a step.
192
-
193
-
194
-
195
- Parameters
196
- ----------
197
- load_policy : str, default: "fresh"
198
- The policy for loading the checkpoint. The following policies are supported:
199
- - "eager": Loads the the latest available checkpoint within the namespace.
200
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
201
- will be loaded at the start of the task.
202
- - "none": Do not load any checkpoint
203
- - "fresh": Loads the lastest checkpoint created within the running Task.
204
- This mode helps loading checkpoints across various retry attempts of the same task.
205
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
206
- created within the task will be loaded when the task is retries execution on failure.
188
+ Specifies that the step will success under all circumstances.
207
189
 
208
- temp_dir_root : str, default: None
209
- The root directory under which `current.checkpoint.directory` will be created.
210
- """
211
- ...
212
-
213
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
214
- """
215
- Specifies that this step should execute on DGX cloud.
190
+ The decorator will create an optional artifact, specified by `var`, which
191
+ contains the exception raised. You can use it to detect the presence
192
+ of errors, indicating that all happy-path artifacts produced by the step
193
+ are missing.
216
194
 
217
195
 
218
196
  Parameters
219
197
  ----------
220
- gpu : int
221
- Number of GPUs to use.
222
- gpu_type : str
223
- Type of Nvidia GPU to use.
224
- queue_timeout : int
225
- Time to keep the job in NVCF's queue.
198
+ var : str, optional, default None
199
+ Name of the artifact in which to store the caught exception.
200
+ If not specified, the exception is not stored.
201
+ print_exception : bool, default True
202
+ Determines whether or not the exception is printed to
203
+ stdout when caught.
226
204
  """
227
205
  ...
228
206
 
229
- def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
207
+ @typing.overload
208
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
230
209
  """
231
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
210
+ Enables loading / saving of models within a step.
232
211
 
233
- User code call
234
- -----------
235
- @ollama(
236
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
237
- backend='local'
238
- )
239
-
240
- Valid backend options
241
- ---------------------
242
- - 'local': Run as a separate process on the local task machine.
243
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
244
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
245
-
246
- Valid model options
247
- ----------------
248
- - 'llama3.2'
249
- - 'llama3.3'
250
- - any model here https://ollama.com/search
251
212
 
252
213
 
253
214
  Parameters
254
215
  ----------
255
- models: list[Ollama]
256
- List of Ollama containers running models in sidecars.
257
- backend: str
258
- Determines where and how to run the Ollama process.
259
- """
260
- ...
261
-
262
- @typing.overload
263
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
264
- """
265
- Creates a human-readable report, a Metaflow Card, after this step completes.
266
-
267
- Note that you may add multiple `@card` decorators in a step with different parameters.
216
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
217
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
218
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
219
+ - `current.checkpoint`
220
+ - `current.model`
221
+ - `current.huggingface_hub`
268
222
 
223
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
224
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
225
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
269
226
 
270
- Parameters
271
- ----------
272
- type : str, default 'default'
273
- Card type.
274
- id : str, optional, default None
275
- If multiple cards are present, use this id to identify this card.
276
- options : Dict[str, Any], default {}
277
- Options passed to the card. The contents depend on the card type.
278
- timeout : int, default 45
279
- Interrupt reporting if it takes more than this many seconds.
227
+ temp_dir_root : str, default: None
228
+ The root directory under which `current.model.loaded` will store loaded models
280
229
  """
281
230
  ...
282
231
 
283
232
  @typing.overload
284
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
233
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
285
234
  ...
286
235
 
287
236
  @typing.overload
288
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
237
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
289
238
  ...
290
239
 
291
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
240
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
292
241
  """
293
- Creates a human-readable report, a Metaflow Card, after this step completes.
242
+ Enables loading / saving of models within a step.
294
243
 
295
- Note that you may add multiple `@card` decorators in a step with different parameters.
296
244
 
297
245
 
298
246
  Parameters
299
247
  ----------
300
- type : str, default 'default'
301
- Card type.
302
- id : str, optional, default None
303
- If multiple cards are present, use this id to identify this card.
304
- options : Dict[str, Any], default {}
305
- Options passed to the card. The contents depend on the card type.
306
- timeout : int, default 45
307
- Interrupt reporting if it takes more than this many seconds.
308
- """
309
- ...
310
-
311
- @typing.overload
312
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
313
- """
314
- Specifies that the step will success under all circumstances.
315
-
316
- The decorator will create an optional artifact, specified by `var`, which
317
- contains the exception raised. You can use it to detect the presence
318
- of errors, indicating that all happy-path artifacts produced by the step
319
- are missing.
248
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
249
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
250
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
251
+ - `current.checkpoint`
252
+ - `current.model`
253
+ - `current.huggingface_hub`
320
254
 
255
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
256
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
257
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
321
258
 
322
- Parameters
323
- ----------
324
- var : str, optional, default None
325
- Name of the artifact in which to store the caught exception.
326
- If not specified, the exception is not stored.
327
- print_exception : bool, default True
328
- Determines whether or not the exception is printed to
329
- stdout when caught.
259
+ temp_dir_root : str, default: None
260
+ The root directory under which `current.model.loaded` will store loaded models
330
261
  """
331
262
  ...
332
263
 
333
- @typing.overload
334
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
335
- ...
336
-
337
- @typing.overload
338
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
339
- ...
340
-
341
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
264
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
342
265
  """
343
- Specifies that the step will success under all circumstances.
344
-
345
- The decorator will create an optional artifact, specified by `var`, which
346
- contains the exception raised. You can use it to detect the presence
347
- of errors, indicating that all happy-path artifacts produced by the step
348
- are missing.
266
+ Specifies that this step should execute on Kubernetes.
349
267
 
350
268
 
351
269
  Parameters
352
270
  ----------
353
- var : str, optional, default None
354
- Name of the artifact in which to store the caught exception.
355
- If not specified, the exception is not stored.
356
- print_exception : bool, default True
357
- Determines whether or not the exception is printed to
358
- stdout when caught.
271
+ cpu : int, default 1
272
+ Number of CPUs required for this step. If `@resources` is
273
+ also present, the maximum value from all decorators is used.
274
+ memory : int, default 4096
275
+ Memory size (in MB) required for this step. If
276
+ `@resources` is also present, the maximum value from all decorators is
277
+ used.
278
+ disk : int, default 10240
279
+ Disk size (in MB) required for this step. If
280
+ `@resources` is also present, the maximum value from all decorators is
281
+ used.
282
+ image : str, optional, default None
283
+ Docker image to use when launching on Kubernetes. If not specified, and
284
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
285
+ not, a default Docker image mapping to the current version of Python is used.
286
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
287
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
288
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
289
+ Kubernetes service account to use when launching pod in Kubernetes.
290
+ secrets : List[str], optional, default None
291
+ Kubernetes secrets to use when launching pod in Kubernetes. These
292
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
293
+ in Metaflow configuration.
294
+ node_selector: Union[Dict[str,str], str], optional, default None
295
+ Kubernetes node selector(s) to apply to the pod running the task.
296
+ Can be passed in as a comma separated string of values e.g.
297
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
298
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
299
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
300
+ Kubernetes namespace to use when launching pod in Kubernetes.
301
+ gpu : int, optional, default None
302
+ Number of GPUs required for this step. A value of zero implies that
303
+ the scheduled node should not have GPUs.
304
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
305
+ The vendor of the GPUs to be used for this step.
306
+ tolerations : List[str], default []
307
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
308
+ Kubernetes tolerations to use when launching pod in Kubernetes.
309
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
310
+ Kubernetes labels to use when launching pod in Kubernetes.
311
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
312
+ Kubernetes annotations to use when launching pod in Kubernetes.
313
+ use_tmpfs : bool, default False
314
+ This enables an explicit tmpfs mount for this step.
315
+ tmpfs_tempdir : bool, default True
316
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
317
+ tmpfs_size : int, optional, default: None
318
+ The value for the size (in MiB) of the tmpfs mount for this step.
319
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
320
+ memory allocated for this step.
321
+ tmpfs_path : str, optional, default /metaflow_temp
322
+ Path to tmpfs mount for this step.
323
+ persistent_volume_claims : Dict[str, str], optional, default None
324
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
325
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
326
+ shared_memory: int, optional
327
+ Shared memory size (in MiB) required for this step
328
+ port: int, optional
329
+ Port number to specify in the Kubernetes job object
330
+ compute_pool : str, optional, default None
331
+ Compute pool to be used for for this step.
332
+ If not specified, any accessible compute pool within the perimeter is used.
333
+ hostname_resolution_timeout: int, default 10 * 60
334
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
335
+ Only applicable when @parallel is used.
336
+ qos: str, default: Burstable
337
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
338
+
339
+ security_context: Dict[str, Any], optional, default None
340
+ Container security context. Applies to the task container. Allows the following keys:
341
+ - privileged: bool, optional, default None
342
+ - allow_privilege_escalation: bool, optional, default None
343
+ - run_as_user: int, optional, default None
344
+ - run_as_group: int, optional, default None
345
+ - run_as_non_root: bool, optional, default None
359
346
  """
360
347
  ...
361
348
 
@@ -418,87 +405,59 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
418
405
  """
419
406
  ...
420
407
 
421
- @typing.overload
422
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
408
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
423
409
  """
424
- Specifies the PyPI packages for the step.
425
-
426
- Information in this decorator will augment any
427
- attributes set in the `@pyi_base` flow-level decorator. Hence,
428
- you can use `@pypi_base` to set packages required by all
429
- steps and use `@pypi` to specify step-specific overrides.
410
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
430
411
 
431
412
 
432
413
  Parameters
433
414
  ----------
434
- packages : Dict[str, str], default: {}
435
- Packages to use for this step. The key is the name of the package
436
- and the value is the version to use.
437
- python : str, optional, default: None
438
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
439
- that the version used will correspond to the version of the Python interpreter used to start the run.
415
+ temp_dir_root : str, optional
416
+ The root directory that will hold the temporary directory where objects will be downloaded.
417
+
418
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
419
+ The list of repos (models/datasets) to load.
420
+
421
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
422
+
423
+ - If repo (model/dataset) is not found in the datastore:
424
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
425
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
426
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
427
+
428
+ - If repo is found in the datastore:
429
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
440
430
  """
441
431
  ...
442
432
 
443
- @typing.overload
444
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
445
- ...
446
-
447
- @typing.overload
448
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
449
- ...
450
-
451
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
433
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
452
434
  """
453
- Specifies the PyPI packages for the step.
454
-
455
- Information in this decorator will augment any
456
- attributes set in the `@pyi_base` flow-level decorator. Hence,
457
- you can use `@pypi_base` to set packages required by all
458
- steps and use `@pypi` to specify step-specific overrides.
435
+ Specifies that this step should execute on DGX cloud.
459
436
 
460
437
 
461
438
  Parameters
462
439
  ----------
463
- packages : Dict[str, str], default: {}
464
- Packages to use for this step. The key is the name of the package
465
- and the value is the version to use.
466
- python : str, optional, default: None
467
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
468
- that the version used will correspond to the version of the Python interpreter used to start the run.
440
+ gpu : int
441
+ Number of GPUs to use.
442
+ gpu_type : str
443
+ Type of Nvidia GPU to use.
469
444
  """
470
445
  ...
471
446
 
472
- def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
447
+ @typing.overload
448
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
473
449
  """
474
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
475
-
476
- User code call
477
- -----------
478
- @nim(
479
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
480
- backend='managed'
481
- )
482
-
483
- Valid backend options
484
- ---------------------
485
- - 'managed': Outerbounds selects a compute provider based on the model.
486
-
487
- Valid model options
488
- ----------------
489
- - 'meta/llama3-8b-instruct': 8B parameter model
490
- - 'meta/llama3-70b-instruct': 70B parameter model
491
- - any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
492
-
493
-
494
- Parameters
495
- ----------
496
- models: list[NIM]
497
- List of NIM containers running models in sidecars.
498
- backend: str
499
- Compute provider to run the NIM container.
500
- queue_timeout : int
501
- Time to keep the job in NVCF's queue.
450
+ Internal decorator to support Fast bakery
451
+ """
452
+ ...
453
+
454
+ @typing.overload
455
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
456
+ ...
457
+
458
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
459
+ """
460
+ Internal decorator to support Fast bakery
502
461
  """
503
462
  ...
504
463
 
@@ -577,24 +536,23 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
577
536
  Memory size (in MB) required for this step.
578
537
  shared_memory : int, optional, default None
579
538
  The value for the size (in MiB) of the /dev/shm volume for this step.
580
- This parameter maps to the `--shm-size` option in Docker.
581
- """
582
- ...
583
-
584
- @typing.overload
585
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
586
- """
587
- Internal decorator to support Fast bakery
539
+ This parameter maps to the `--shm-size` option in Docker.
588
540
  """
589
541
  ...
590
542
 
591
- @typing.overload
592
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
593
- ...
594
-
595
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
543
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
596
544
  """
597
- Internal decorator to support Fast bakery
545
+ Specifies that this step should execute on DGX cloud.
546
+
547
+
548
+ Parameters
549
+ ----------
550
+ gpu : int
551
+ Number of GPUs to use.
552
+ gpu_type : str
553
+ Type of Nvidia GPU to use.
554
+ queue_timeout : int
555
+ Time to keep the job in NVCF's queue.
598
556
  """
599
557
  ...
600
558
 
@@ -631,114 +589,95 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
631
589
  """
632
590
  ...
633
591
 
634
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
592
+ def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
635
593
  """
636
- Specifies that this step is used to deploy an instance of the app.
637
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
638
-
594
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
639
595
 
640
- Parameters
641
- ----------
642
- app_port : int
643
- Number of GPUs to use.
644
- app_name : str
645
- Name of the app to deploy.
646
- """
647
- ...
648
-
649
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
650
- """
651
- Specifies that this step should execute on DGX cloud.
596
+ User code call
597
+ -----------
598
+ @ollama(
599
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
600
+ backend='local'
601
+ )
652
602
 
603
+ Valid backend options
604
+ ---------------------
605
+ - 'local': Run as a separate process on the local task machine.
606
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
607
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
653
608
 
654
- Parameters
655
- ----------
656
- gpu : int
657
- Number of GPUs to use.
658
- gpu_type : str
659
- Type of Nvidia GPU to use.
660
- """
661
- ...
662
-
663
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
664
- """
665
- Decorator that helps cache, version and store models/datasets from huggingface hub.
609
+ Valid model options
610
+ ----------------
611
+ - 'llama3.2'
612
+ - 'llama3.3'
613
+ - any model here https://ollama.com/search
666
614
 
667
615
 
668
616
  Parameters
669
617
  ----------
670
- temp_dir_root : str, optional
671
- The root directory that will hold the temporary directory where objects will be downloaded.
672
-
673
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
674
- The list of repos (models/datasets) to load.
675
-
676
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
677
-
678
- - If repo (model/dataset) is not found in the datastore:
679
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
680
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
681
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
682
-
683
- - If repo is found in the datastore:
684
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
618
+ models: list[Ollama]
619
+ List of Ollama containers running models in sidecars.
620
+ backend: str
621
+ Determines where and how to run the Ollama process.
685
622
  """
686
623
  ...
687
624
 
688
625
  @typing.overload
689
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
626
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
690
627
  """
691
- Enables loading / saving of models within a step.
628
+ Specifies the Conda environment for the step.
692
629
 
630
+ Information in this decorator will augment any
631
+ attributes set in the `@conda_base` flow-level decorator. Hence,
632
+ you can use `@conda_base` to set packages required by all
633
+ steps and use `@conda` to specify step-specific overrides.
693
634
 
694
635
 
695
636
  Parameters
696
637
  ----------
697
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
698
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
699
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
700
- - `current.checkpoint`
701
- - `current.model`
702
- - `current.huggingface_hub`
703
-
704
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
705
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
706
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
707
-
708
- temp_dir_root : str, default: None
709
- The root directory under which `current.model.loaded` will store loaded models
638
+ packages : Dict[str, str], default {}
639
+ Packages to use for this step. The key is the name of the package
640
+ and the value is the version to use.
641
+ libraries : Dict[str, str], default {}
642
+ Supported for backward compatibility. When used with packages, packages will take precedence.
643
+ python : str, optional, default None
644
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
645
+ that the version used will correspond to the version of the Python interpreter used to start the run.
646
+ disabled : bool, default False
647
+ If set to True, disables @conda.
710
648
  """
711
649
  ...
712
650
 
713
651
  @typing.overload
714
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
652
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
715
653
  ...
716
654
 
717
655
  @typing.overload
718
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
656
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
719
657
  ...
720
658
 
721
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
659
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
722
660
  """
723
- Enables loading / saving of models within a step.
661
+ Specifies the Conda environment for the step.
724
662
 
663
+ Information in this decorator will augment any
664
+ attributes set in the `@conda_base` flow-level decorator. Hence,
665
+ you can use `@conda_base` to set packages required by all
666
+ steps and use `@conda` to specify step-specific overrides.
725
667
 
726
668
 
727
669
  Parameters
728
670
  ----------
729
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
730
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
731
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
732
- - `current.checkpoint`
733
- - `current.model`
734
- - `current.huggingface_hub`
735
-
736
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
737
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
738
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
739
-
740
- temp_dir_root : str, default: None
741
- The root directory under which `current.model.loaded` will store loaded models
671
+ packages : Dict[str, str], default {}
672
+ Packages to use for this step. The key is the name of the package
673
+ and the value is the version to use.
674
+ libraries : Dict[str, str], default {}
675
+ Supported for backward compatibility. When used with packages, packages will take precedence.
676
+ python : str, optional, default None
677
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
678
+ that the version used will correspond to the version of the Python interpreter used to start the run.
679
+ disabled : bool, default False
680
+ If set to True, disables @conda.
742
681
  """
743
682
  ...
744
683
 
@@ -777,107 +716,109 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
777
716
  """
778
717
  ...
779
718
 
780
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
719
+ @typing.overload
720
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
781
721
  """
782
- Specifies that this step should execute on Kubernetes.
722
+ Enables checkpointing for a step.
723
+
783
724
 
784
725
 
785
726
  Parameters
786
727
  ----------
787
- cpu : int, default 1
788
- Number of CPUs required for this step. If `@resources` is
789
- also present, the maximum value from all decorators is used.
790
- memory : int, default 4096
791
- Memory size (in MB) required for this step. If
792
- `@resources` is also present, the maximum value from all decorators is
793
- used.
794
- disk : int, default 10240
795
- Disk size (in MB) required for this step. If
796
- `@resources` is also present, the maximum value from all decorators is
797
- used.
798
- image : str, optional, default None
799
- Docker image to use when launching on Kubernetes. If not specified, and
800
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
801
- not, a default Docker image mapping to the current version of Python is used.
802
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
803
- If given, the imagePullPolicy to be applied to the Docker image of the step.
804
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
805
- Kubernetes service account to use when launching pod in Kubernetes.
806
- secrets : List[str], optional, default None
807
- Kubernetes secrets to use when launching pod in Kubernetes. These
808
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
809
- in Metaflow configuration.
810
- node_selector: Union[Dict[str,str], str], optional, default None
811
- Kubernetes node selector(s) to apply to the pod running the task.
812
- Can be passed in as a comma separated string of values e.g.
813
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
814
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
815
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
816
- Kubernetes namespace to use when launching pod in Kubernetes.
817
- gpu : int, optional, default None
818
- Number of GPUs required for this step. A value of zero implies that
819
- the scheduled node should not have GPUs.
820
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
821
- The vendor of the GPUs to be used for this step.
822
- tolerations : List[str], default []
823
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
824
- Kubernetes tolerations to use when launching pod in Kubernetes.
825
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
826
- Kubernetes labels to use when launching pod in Kubernetes.
827
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
828
- Kubernetes annotations to use when launching pod in Kubernetes.
829
- use_tmpfs : bool, default False
830
- This enables an explicit tmpfs mount for this step.
831
- tmpfs_tempdir : bool, default True
832
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
833
- tmpfs_size : int, optional, default: None
834
- The value for the size (in MiB) of the tmpfs mount for this step.
835
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
836
- memory allocated for this step.
837
- tmpfs_path : str, optional, default /metaflow_temp
838
- Path to tmpfs mount for this step.
839
- persistent_volume_claims : Dict[str, str], optional, default None
840
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
841
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
842
- shared_memory: int, optional
843
- Shared memory size (in MiB) required for this step
844
- port: int, optional
845
- Port number to specify in the Kubernetes job object
846
- compute_pool : str, optional, default None
847
- Compute pool to be used for for this step.
848
- If not specified, any accessible compute pool within the perimeter is used.
849
- hostname_resolution_timeout: int, default 10 * 60
850
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
851
- Only applicable when @parallel is used.
852
- qos: str, default: Burstable
853
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
728
+ load_policy : str, default: "fresh"
729
+ The policy for loading the checkpoint. The following policies are supported:
730
+ - "eager": Loads the the latest available checkpoint within the namespace.
731
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
732
+ will be loaded at the start of the task.
733
+ - "none": Do not load any checkpoint
734
+ - "fresh": Loads the lastest checkpoint created within the running Task.
735
+ This mode helps loading checkpoints across various retry attempts of the same task.
736
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
737
+ created within the task will be loaded when the task is retries execution on failure.
738
+
739
+ temp_dir_root : str, default: None
740
+ The root directory under which `current.checkpoint.directory` will be created.
741
+ """
742
+ ...
743
+
744
+ @typing.overload
745
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
746
+ ...
747
+
748
+ @typing.overload
749
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
750
+ ...
751
+
752
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
753
+ """
754
+ Enables checkpointing for a step.
755
+
756
+
757
+
758
+ Parameters
759
+ ----------
760
+ load_policy : str, default: "fresh"
761
+ The policy for loading the checkpoint. The following policies are supported:
762
+ - "eager": Loads the the latest available checkpoint within the namespace.
763
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
764
+ will be loaded at the start of the task.
765
+ - "none": Do not load any checkpoint
766
+ - "fresh": Loads the lastest checkpoint created within the running Task.
767
+ This mode helps loading checkpoints across various retry attempts of the same task.
768
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
769
+ created within the task will be loaded when the task is retries execution on failure.
854
770
 
855
- security_context: Dict[str, Any], optional, default None
856
- Container security context. Applies to the task container. Allows the following keys:
857
- - privileged: bool, optional, default None
858
- - allow_privilege_escalation: bool, optional, default None
859
- - run_as_user: int, optional, default None
860
- - run_as_group: int, optional, default None
861
- - run_as_non_root: bool, optional, default None
771
+ temp_dir_root : str, default: None
772
+ The root directory under which `current.checkpoint.directory` will be created.
862
773
  """
863
774
  ...
864
775
 
865
776
  @typing.overload
866
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
777
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
867
778
  """
868
- Decorator prototype for all step decorators. This function gets specialized
869
- and imported for all decorators types by _import_plugin_decorators().
779
+ Creates a human-readable report, a Metaflow Card, after this step completes.
780
+
781
+ Note that you may add multiple `@card` decorators in a step with different parameters.
782
+
783
+
784
+ Parameters
785
+ ----------
786
+ type : str, default 'default'
787
+ Card type.
788
+ id : str, optional, default None
789
+ If multiple cards are present, use this id to identify this card.
790
+ options : Dict[str, Any], default {}
791
+ Options passed to the card. The contents depend on the card type.
792
+ timeout : int, default 45
793
+ Interrupt reporting if it takes more than this many seconds.
870
794
  """
871
795
  ...
872
796
 
873
797
  @typing.overload
874
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
798
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
875
799
  ...
876
800
 
877
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
801
+ @typing.overload
802
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
803
+ ...
804
+
805
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
878
806
  """
879
- Decorator prototype for all step decorators. This function gets specialized
880
- and imported for all decorators types by _import_plugin_decorators().
807
+ Creates a human-readable report, a Metaflow Card, after this step completes.
808
+
809
+ Note that you may add multiple `@card` decorators in a step with different parameters.
810
+
811
+
812
+ Parameters
813
+ ----------
814
+ type : str, default 'default'
815
+ Card type.
816
+ id : str, optional, default None
817
+ If multiple cards are present, use this id to identify this card.
818
+ options : Dict[str, Any], default {}
819
+ Options passed to the card. The contents depend on the card type.
820
+ timeout : int, default 45
821
+ Interrupt reporting if it takes more than this many seconds.
881
822
  """
882
823
  ...
883
824
 
@@ -937,61 +878,87 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
937
878
  ...
938
879
 
939
880
  @typing.overload
940
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
881
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
941
882
  """
942
- Specifies the Conda environment for the step.
883
+ Specifies the PyPI packages for the step.
943
884
 
944
885
  Information in this decorator will augment any
945
- attributes set in the `@conda_base` flow-level decorator. Hence,
946
- you can use `@conda_base` to set packages required by all
947
- steps and use `@conda` to specify step-specific overrides.
886
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
887
+ you can use `@pypi_base` to set packages required by all
888
+ steps and use `@pypi` to specify step-specific overrides.
948
889
 
949
890
 
950
891
  Parameters
951
892
  ----------
952
- packages : Dict[str, str], default {}
893
+ packages : Dict[str, str], default: {}
953
894
  Packages to use for this step. The key is the name of the package
954
895
  and the value is the version to use.
955
- libraries : Dict[str, str], default {}
956
- Supported for backward compatibility. When used with packages, packages will take precedence.
957
- python : str, optional, default None
896
+ python : str, optional, default: None
958
897
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
959
898
  that the version used will correspond to the version of the Python interpreter used to start the run.
960
- disabled : bool, default False
961
- If set to True, disables @conda.
962
899
  """
963
900
  ...
964
901
 
965
902
  @typing.overload
966
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
903
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
967
904
  ...
968
905
 
969
906
  @typing.overload
970
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
907
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
971
908
  ...
972
909
 
973
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
910
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
974
911
  """
975
- Specifies the Conda environment for the step.
912
+ Specifies the PyPI packages for the step.
976
913
 
977
914
  Information in this decorator will augment any
978
- attributes set in the `@conda_base` flow-level decorator. Hence,
979
- you can use `@conda_base` to set packages required by all
980
- steps and use `@conda` to specify step-specific overrides.
915
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
916
+ you can use `@pypi_base` to set packages required by all
917
+ steps and use `@pypi` to specify step-specific overrides.
981
918
 
982
919
 
983
920
  Parameters
984
921
  ----------
985
- packages : Dict[str, str], default {}
922
+ packages : Dict[str, str], default: {}
986
923
  Packages to use for this step. The key is the name of the package
987
924
  and the value is the version to use.
988
- libraries : Dict[str, str], default {}
989
- Supported for backward compatibility. When used with packages, packages will take precedence.
990
- python : str, optional, default None
925
+ python : str, optional, default: None
991
926
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
992
927
  that the version used will correspond to the version of the Python interpreter used to start the run.
993
- disabled : bool, default False
994
- If set to True, disables @conda.
928
+ """
929
+ ...
930
+
931
+ @typing.overload
932
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
933
+ """
934
+ Decorator prototype for all step decorators. This function gets specialized
935
+ and imported for all decorators types by _import_plugin_decorators().
936
+ """
937
+ ...
938
+
939
+ @typing.overload
940
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
941
+ ...
942
+
943
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
944
+ """
945
+ Decorator prototype for all step decorators. This function gets specialized
946
+ and imported for all decorators types by _import_plugin_decorators().
947
+ """
948
+ ...
949
+
950
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
951
+ """
952
+ Specifies that this step is used to deploy an instance of the app.
953
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
954
+
955
+
956
+ Parameters
957
+ ----------
958
+ app_port : int
959
+ Number of GPUs to use.
960
+ app_name : str
961
+ Name of the app to deploy.
995
962
  """
996
963
  ...
997
964
 
@@ -1088,97 +1055,130 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1088
1055
  """
1089
1056
  ...
1090
1057
 
1091
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1058
+ @typing.overload
1059
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1092
1060
  """
1093
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1094
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1061
+ Specifies the Conda environment for all steps of the flow.
1062
+
1063
+ Use `@conda_base` to set common libraries required by all
1064
+ steps and use `@conda` to specify step-specific additions.
1095
1065
 
1096
1066
 
1097
1067
  Parameters
1098
1068
  ----------
1099
- timeout : int
1100
- Time, in seconds before the task times out and fails. (Default: 3600)
1101
- poke_interval : int
1102
- Time in seconds that the job should wait in between each try. (Default: 60)
1103
- mode : str
1104
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1105
- exponential_backoff : bool
1106
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1107
- pool : str
1108
- the slot pool this task should run in,
1109
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1110
- soft_fail : bool
1111
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1112
- name : str
1113
- Name of the sensor on Airflow
1114
- description : str
1115
- Description of sensor in the Airflow UI
1116
- external_dag_id : str
1117
- The dag_id that contains the task you want to wait for.
1118
- external_task_ids : List[str]
1119
- The list of task_ids that you want to wait for.
1120
- If None (default value) the sensor waits for the DAG. (Default: None)
1121
- allowed_states : List[str]
1122
- Iterable of allowed states, (Default: ['success'])
1123
- failed_states : List[str]
1124
- Iterable of failed or dis-allowed states. (Default: None)
1125
- execution_delta : datetime.timedelta
1126
- time difference with the previous execution to look at,
1127
- the default is the same logical date as the current task or DAG. (Default: None)
1128
- check_existence: bool
1129
- Set to True to check if the external task exists or check if
1130
- the DAG to wait for exists. (Default: True)
1069
+ packages : Dict[str, str], default {}
1070
+ Packages to use for this flow. The key is the name of the package
1071
+ and the value is the version to use.
1072
+ libraries : Dict[str, str], default {}
1073
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1074
+ python : str, optional, default None
1075
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1076
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1077
+ disabled : bool, default False
1078
+ If set to True, disables Conda.
1131
1079
  """
1132
1080
  ...
1133
1081
 
1134
1082
  @typing.overload
1135
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1083
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1084
+ ...
1085
+
1086
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1136
1087
  """
1137
- Specifies the times when the flow should be run when running on a
1138
- production scheduler.
1088
+ Specifies the Conda environment for all steps of the flow.
1089
+
1090
+ Use `@conda_base` to set common libraries required by all
1091
+ steps and use `@conda` to specify step-specific additions.
1139
1092
 
1140
1093
 
1141
1094
  Parameters
1142
1095
  ----------
1143
- hourly : bool, default False
1144
- Run the workflow hourly.
1145
- daily : bool, default True
1146
- Run the workflow daily.
1147
- weekly : bool, default False
1148
- Run the workflow weekly.
1149
- cron : str, optional, default None
1150
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1151
- specified by this expression.
1152
- timezone : str, optional, default None
1153
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1154
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1096
+ packages : Dict[str, str], default {}
1097
+ Packages to use for this flow. The key is the name of the package
1098
+ and the value is the version to use.
1099
+ libraries : Dict[str, str], default {}
1100
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1101
+ python : str, optional, default None
1102
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1103
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1104
+ disabled : bool, default False
1105
+ If set to True, disables Conda.
1106
+ """
1107
+ ...
1108
+
1109
+ @typing.overload
1110
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1111
+ """
1112
+ Specifies the PyPI packages for all steps of the flow.
1113
+
1114
+ Use `@pypi_base` to set common packages required by all
1115
+ steps and use `@pypi` to specify step-specific overrides.
1116
+
1117
+ Parameters
1118
+ ----------
1119
+ packages : Dict[str, str], default: {}
1120
+ Packages to use for this flow. The key is the name of the package
1121
+ and the value is the version to use.
1122
+ python : str, optional, default: None
1123
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1124
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1125
+ """
1126
+ ...
1127
+
1128
+ @typing.overload
1129
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1130
+ ...
1131
+
1132
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1133
+ """
1134
+ Specifies the PyPI packages for all steps of the flow.
1135
+
1136
+ Use `@pypi_base` to set common packages required by all
1137
+ steps and use `@pypi` to specify step-specific overrides.
1138
+
1139
+ Parameters
1140
+ ----------
1141
+ packages : Dict[str, str], default: {}
1142
+ Packages to use for this flow. The key is the name of the package
1143
+ and the value is the version to use.
1144
+ python : str, optional, default: None
1145
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1146
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1155
1147
  """
1156
1148
  ...
1157
1149
 
1158
- @typing.overload
1159
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1160
- ...
1161
-
1162
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1150
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1163
1151
  """
1164
- Specifies the times when the flow should be run when running on a
1165
- production scheduler.
1152
+ Specifies what flows belong to the same project.
1153
+
1154
+ A project-specific namespace is created for all flows that
1155
+ use the same `@project(name)`.
1166
1156
 
1167
1157
 
1168
1158
  Parameters
1169
1159
  ----------
1170
- hourly : bool, default False
1171
- Run the workflow hourly.
1172
- daily : bool, default True
1173
- Run the workflow daily.
1174
- weekly : bool, default False
1175
- Run the workflow weekly.
1176
- cron : str, optional, default None
1177
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1178
- specified by this expression.
1179
- timezone : str, optional, default None
1180
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1181
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1160
+ name : str
1161
+ Project name. Make sure that the name is unique amongst all
1162
+ projects that use the same production scheduler. The name may
1163
+ contain only lowercase alphanumeric characters and underscores.
1164
+
1165
+ branch : Optional[str], default None
1166
+ The branch to use. If not specified, the branch is set to
1167
+ `user.<username>` unless `production` is set to `True`. This can
1168
+ also be set on the command line using `--branch` as a top-level option.
1169
+ It is an error to specify `branch` in the decorator and on the command line.
1170
+
1171
+ production : bool, default False
1172
+ Whether or not the branch is the production branch. This can also be set on the
1173
+ command line using `--production` as a top-level option. It is an error to specify
1174
+ `production` in the decorator and on the command line.
1175
+ The project branch name will be:
1176
+ - if `branch` is specified:
1177
+ - if `production` is True: `prod.<branch>`
1178
+ - if `production` is False: `test.<branch>`
1179
+ - if `branch` is not specified:
1180
+ - if `production` is True: `prod`
1181
+ - if `production` is False: `user.<username>`
1182
1182
  """
1183
1183
  ...
1184
1184
 
@@ -1225,54 +1225,46 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1225
1225
  """
1226
1226
  ...
1227
1227
 
1228
- @typing.overload
1229
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1230
- """
1231
- Specifies the Conda environment for all steps of the flow.
1232
-
1233
- Use `@conda_base` to set common libraries required by all
1234
- steps and use `@conda` to specify step-specific additions.
1235
-
1236
-
1237
- Parameters
1238
- ----------
1239
- packages : Dict[str, str], default {}
1240
- Packages to use for this flow. The key is the name of the package
1241
- and the value is the version to use.
1242
- libraries : Dict[str, str], default {}
1243
- Supported for backward compatibility. When used with packages, packages will take precedence.
1244
- python : str, optional, default None
1245
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1246
- that the version used will correspond to the version of the Python interpreter used to start the run.
1247
- disabled : bool, default False
1248
- If set to True, disables Conda.
1249
- """
1250
- ...
1251
-
1252
- @typing.overload
1253
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1254
- ...
1255
-
1256
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1228
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1257
1229
  """
1258
- Specifies the Conda environment for all steps of the flow.
1259
-
1260
- Use `@conda_base` to set common libraries required by all
1261
- steps and use `@conda` to specify step-specific additions.
1230
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1231
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1262
1232
 
1263
1233
 
1264
1234
  Parameters
1265
1235
  ----------
1266
- packages : Dict[str, str], default {}
1267
- Packages to use for this flow. The key is the name of the package
1268
- and the value is the version to use.
1269
- libraries : Dict[str, str], default {}
1270
- Supported for backward compatibility. When used with packages, packages will take precedence.
1271
- python : str, optional, default None
1272
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1273
- that the version used will correspond to the version of the Python interpreter used to start the run.
1274
- disabled : bool, default False
1275
- If set to True, disables Conda.
1236
+ timeout : int
1237
+ Time, in seconds before the task times out and fails. (Default: 3600)
1238
+ poke_interval : int
1239
+ Time in seconds that the job should wait in between each try. (Default: 60)
1240
+ mode : str
1241
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1242
+ exponential_backoff : bool
1243
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1244
+ pool : str
1245
+ the slot pool this task should run in,
1246
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1247
+ soft_fail : bool
1248
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1249
+ name : str
1250
+ Name of the sensor on Airflow
1251
+ description : str
1252
+ Description of sensor in the Airflow UI
1253
+ external_dag_id : str
1254
+ The dag_id that contains the task you want to wait for.
1255
+ external_task_ids : List[str]
1256
+ The list of task_ids that you want to wait for.
1257
+ If None (default value) the sensor waits for the DAG. (Default: None)
1258
+ allowed_states : List[str]
1259
+ Iterable of allowed states, (Default: ['success'])
1260
+ failed_states : List[str]
1261
+ Iterable of failed or dis-allowed states. (Default: None)
1262
+ execution_delta : datetime.timedelta
1263
+ time difference with the previous execution to look at,
1264
+ the default is the same logical date as the current task or DAG. (Default: None)
1265
+ check_existence: bool
1266
+ Set to True to check if the external task exists or check if
1267
+ the DAG to wait for exists. (Default: True)
1276
1268
  """
1277
1269
  ...
1278
1270
 
@@ -1377,6 +1369,57 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1377
1369
  """
1378
1370
  ...
1379
1371
 
1372
+ @typing.overload
1373
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1374
+ """
1375
+ Specifies the times when the flow should be run when running on a
1376
+ production scheduler.
1377
+
1378
+
1379
+ Parameters
1380
+ ----------
1381
+ hourly : bool, default False
1382
+ Run the workflow hourly.
1383
+ daily : bool, default True
1384
+ Run the workflow daily.
1385
+ weekly : bool, default False
1386
+ Run the workflow weekly.
1387
+ cron : str, optional, default None
1388
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1389
+ specified by this expression.
1390
+ timezone : str, optional, default None
1391
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1392
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1393
+ """
1394
+ ...
1395
+
1396
+ @typing.overload
1397
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1398
+ ...
1399
+
1400
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1401
+ """
1402
+ Specifies the times when the flow should be run when running on a
1403
+ production scheduler.
1404
+
1405
+
1406
+ Parameters
1407
+ ----------
1408
+ hourly : bool, default False
1409
+ Run the workflow hourly.
1410
+ daily : bool, default True
1411
+ Run the workflow daily.
1412
+ weekly : bool, default False
1413
+ Run the workflow weekly.
1414
+ cron : str, optional, default None
1415
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1416
+ specified by this expression.
1417
+ timezone : str, optional, default None
1418
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1419
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1420
+ """
1421
+ ...
1422
+
1380
1423
  def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1381
1424
  """
1382
1425
  Allows setting external datastores to save data for the
@@ -1491,81 +1534,5 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1491
1534
  """
1492
1535
  ...
1493
1536
 
1494
- @typing.overload
1495
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1496
- """
1497
- Specifies the PyPI packages for all steps of the flow.
1498
-
1499
- Use `@pypi_base` to set common packages required by all
1500
- steps and use `@pypi` to specify step-specific overrides.
1501
-
1502
- Parameters
1503
- ----------
1504
- packages : Dict[str, str], default: {}
1505
- Packages to use for this flow. The key is the name of the package
1506
- and the value is the version to use.
1507
- python : str, optional, default: None
1508
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1509
- that the version used will correspond to the version of the Python interpreter used to start the run.
1510
- """
1511
- ...
1512
-
1513
- @typing.overload
1514
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1515
- ...
1516
-
1517
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1518
- """
1519
- Specifies the PyPI packages for all steps of the flow.
1520
-
1521
- Use `@pypi_base` to set common packages required by all
1522
- steps and use `@pypi` to specify step-specific overrides.
1523
-
1524
- Parameters
1525
- ----------
1526
- packages : Dict[str, str], default: {}
1527
- Packages to use for this flow. The key is the name of the package
1528
- and the value is the version to use.
1529
- python : str, optional, default: None
1530
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1531
- that the version used will correspond to the version of the Python interpreter used to start the run.
1532
- """
1533
- ...
1534
-
1535
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1536
- """
1537
- Specifies what flows belong to the same project.
1538
-
1539
- A project-specific namespace is created for all flows that
1540
- use the same `@project(name)`.
1541
-
1542
-
1543
- Parameters
1544
- ----------
1545
- name : str
1546
- Project name. Make sure that the name is unique amongst all
1547
- projects that use the same production scheduler. The name may
1548
- contain only lowercase alphanumeric characters and underscores.
1549
-
1550
- branch : Optional[str], default None
1551
- The branch to use. If not specified, the branch is set to
1552
- `user.<username>` unless `production` is set to `True`. This can
1553
- also be set on the command line using `--branch` as a top-level option.
1554
- It is an error to specify `branch` in the decorator and on the command line.
1555
-
1556
- production : bool, default False
1557
- Whether or not the branch is the production branch. This can also be set on the
1558
- command line using `--production` as a top-level option. It is an error to specify
1559
- `production` in the decorator and on the command line.
1560
- The project branch name will be:
1561
- - if `branch` is specified:
1562
- - if `production` is True: `prod.<branch>`
1563
- - if `production` is False: `test.<branch>`
1564
- - if `branch` is not specified:
1565
- - if `production` is True: `prod`
1566
- - if `production` is False: `user.<username>`
1567
- """
1568
- ...
1569
-
1570
1537
  pkg_name: str
1571
1538