ob-metaflow-stubs 6.0.3.168rc0__py2.py3-none-any.whl → 6.0.3.170__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. metaflow-stubs/__init__.pyi +646 -679
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +3 -3
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +86 -86
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +4 -4
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  80. metaflow-stubs/multicore_utils.pyi +2 -2
  81. metaflow-stubs/parameters.pyi +3 -3
  82. metaflow-stubs/plugins/__init__.pyi +9 -9
  83. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  84. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  85. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  86. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  88. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  89. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  90. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  92. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  93. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  94. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  95. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  96. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  97. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  99. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  100. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  102. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  103. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  105. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  106. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  108. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  109. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  110. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  111. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  112. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  113. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  114. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  115. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  116. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  117. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  118. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  119. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  120. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  121. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  122. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  123. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  124. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  126. metaflow-stubs/plugins/cards/card_modules/basic.pyi +4 -4
  127. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  128. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  129. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  130. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  131. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  132. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  133. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  134. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  135. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  136. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  138. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  139. metaflow-stubs/plugins/datatools/s3/s3.pyi +2 -2
  140. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  141. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  142. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  143. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  144. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  145. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  146. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  147. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  148. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  150. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  151. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  152. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  153. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  154. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  155. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  156. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  157. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  158. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  159. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  160. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  161. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  162. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  163. metaflow-stubs/plugins/perimeters.pyi +2 -2
  164. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  165. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  166. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  167. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  168. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  169. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  170. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  171. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  172. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  173. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  174. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  175. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  176. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  177. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  178. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  179. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  180. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  181. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  182. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  183. metaflow-stubs/profilers/__init__.pyi +2 -2
  184. metaflow-stubs/pylint_wrapper.pyi +2 -2
  185. metaflow-stubs/runner/__init__.pyi +2 -2
  186. metaflow-stubs/runner/deployer.pyi +29 -29
  187. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  188. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  189. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  190. metaflow-stubs/runner/nbrun.pyi +2 -2
  191. metaflow-stubs/runner/subprocess_manager.pyi +3 -3
  192. metaflow-stubs/runner/utils.pyi +2 -2
  193. metaflow-stubs/system/__init__.pyi +2 -2
  194. metaflow-stubs/system/system_logger.pyi +3 -3
  195. metaflow-stubs/system/system_monitor.pyi +2 -2
  196. metaflow-stubs/tagging_util.pyi +2 -2
  197. metaflow-stubs/tuple_util.pyi +2 -2
  198. metaflow-stubs/user_configs/__init__.pyi +2 -2
  199. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  200. metaflow-stubs/user_configs/config_options.pyi +2 -2
  201. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  202. {ob_metaflow_stubs-6.0.3.168rc0.dist-info → ob_metaflow_stubs-6.0.3.170.dist-info}/METADATA +1 -1
  203. ob_metaflow_stubs-6.0.3.170.dist-info/RECORD +206 -0
  204. ob_metaflow_stubs-6.0.3.168rc0.dist-info/RECORD +0 -206
  205. {ob_metaflow_stubs-6.0.3.168rc0.dist-info → ob_metaflow_stubs-6.0.3.170.dist-info}/WHEEL +0 -0
  206. {ob_metaflow_stubs-6.0.3.168rc0.dist-info → ob_metaflow_stubs-6.0.3.170.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.11.2+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-05-19T23:34:02.941331 #
3
+ # MF version: 2.15.14.1+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-05-22T07:40:40.319392 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -37,16 +37,16 @@ from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDec
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
38
  from . import tuple_util as tuple_util
39
39
  from . import cards as cards
40
- from . import metaflow_git as metaflow_git
41
40
  from . import events as events
41
+ from . import metaflow_git as metaflow_git
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
+ from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
47
48
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
48
49
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
49
- from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
52
52
  from .client.core import get_namespace as get_namespace
@@ -153,109 +153,61 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
153
153
  """
154
154
  ...
155
155
 
156
- @typing.overload
157
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
156
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
158
157
  """
159
- Enables checkpointing for a step.
160
-
158
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
161
159
 
162
160
 
163
161
  Parameters
164
162
  ----------
165
- load_policy : str, default: "fresh"
166
- The policy for loading the checkpoint. The following policies are supported:
167
- - "eager": Loads the the latest available checkpoint within the namespace.
168
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
169
- will be loaded at the start of the task.
170
- - "none": Do not load any checkpoint
171
- - "fresh": Loads the lastest checkpoint created within the running Task.
172
- This mode helps loading checkpoints across various retry attempts of the same task.
173
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
174
- created within the task will be loaded when the task is retries execution on failure.
175
-
176
- temp_dir_root : str, default: None
177
- The root directory under which `current.checkpoint.directory` will be created.
178
- """
179
- ...
180
-
181
- @typing.overload
182
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
183
- ...
184
-
185
- @typing.overload
186
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
187
- ...
188
-
189
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
190
- """
191
- Enables checkpointing for a step.
163
+ temp_dir_root : str, optional
164
+ The root directory that will hold the temporary directory where objects will be downloaded.
192
165
 
166
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
167
+ The list of repos (models/datasets) to load.
193
168
 
169
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
194
170
 
195
- Parameters
196
- ----------
197
- load_policy : str, default: "fresh"
198
- The policy for loading the checkpoint. The following policies are supported:
199
- - "eager": Loads the the latest available checkpoint within the namespace.
200
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
201
- will be loaded at the start of the task.
202
- - "none": Do not load any checkpoint
203
- - "fresh": Loads the lastest checkpoint created within the running Task.
204
- This mode helps loading checkpoints across various retry attempts of the same task.
205
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
206
- created within the task will be loaded when the task is retries execution on failure.
171
+ - If repo (model/dataset) is not found in the datastore:
172
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
173
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
174
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
207
175
 
208
- temp_dir_root : str, default: None
209
- The root directory under which `current.checkpoint.directory` will be created.
176
+ - If repo is found in the datastore:
177
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
210
178
  """
211
179
  ...
212
180
 
213
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
181
+ @typing.overload
182
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
214
183
  """
215
- Specifies that this step should execute on DGX cloud.
184
+ Specifies environment variables to be set prior to the execution of a step.
216
185
 
217
186
 
218
187
  Parameters
219
188
  ----------
220
- gpu : int
221
- Number of GPUs to use.
222
- gpu_type : str
223
- Type of Nvidia GPU to use.
224
- queue_timeout : int
225
- Time to keep the job in NVCF's queue.
189
+ vars : Dict[str, str], default {}
190
+ Dictionary of environment variables to set.
226
191
  """
227
192
  ...
228
193
 
229
- def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
194
+ @typing.overload
195
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
196
+ ...
197
+
198
+ @typing.overload
199
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
200
+ ...
201
+
202
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
230
203
  """
231
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
232
-
233
- User code call
234
- -----------
235
- @ollama(
236
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
237
- backend='local'
238
- )
239
-
240
- Valid backend options
241
- ---------------------
242
- - 'local': Run as a separate process on the local task machine.
243
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
244
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
245
-
246
- Valid model options
247
- ----------------
248
- - 'llama3.2'
249
- - 'llama3.3'
250
- - any model here https://ollama.com/search
204
+ Specifies environment variables to be set prior to the execution of a step.
251
205
 
252
206
 
253
207
  Parameters
254
208
  ----------
255
- models: list[Ollama]
256
- List of Ollama containers running models in sidecars.
257
- backend: str
258
- Determines where and how to run the Ollama process.
209
+ vars : Dict[str, str], default {}
210
+ Dictionary of environment variables to set.
259
211
  """
260
212
  ...
261
213
 
@@ -308,54 +260,111 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
308
260
  """
309
261
  ...
310
262
 
263
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
264
+ """
265
+ Specifies that this step should execute on DGX cloud.
266
+
267
+
268
+ Parameters
269
+ ----------
270
+ gpu : int
271
+ Number of GPUs to use.
272
+ gpu_type : str
273
+ Type of Nvidia GPU to use.
274
+ queue_timeout : int
275
+ Time to keep the job in NVCF's queue.
276
+ """
277
+ ...
278
+
311
279
  @typing.overload
312
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
280
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
313
281
  """
314
- Specifies that the step will success under all circumstances.
282
+ Enables loading / saving of models within a step.
315
283
 
316
- The decorator will create an optional artifact, specified by `var`, which
317
- contains the exception raised. You can use it to detect the presence
318
- of errors, indicating that all happy-path artifacts produced by the step
319
- are missing.
320
284
 
321
285
 
322
286
  Parameters
323
287
  ----------
324
- var : str, optional, default None
325
- Name of the artifact in which to store the caught exception.
326
- If not specified, the exception is not stored.
327
- print_exception : bool, default True
328
- Determines whether or not the exception is printed to
329
- stdout when caught.
288
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
289
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
290
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
291
+ - `current.checkpoint`
292
+ - `current.model`
293
+ - `current.huggingface_hub`
294
+
295
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
296
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
297
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
298
+
299
+ temp_dir_root : str, default: None
300
+ The root directory under which `current.model.loaded` will store loaded models
330
301
  """
331
302
  ...
332
303
 
333
304
  @typing.overload
334
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
305
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
335
306
  ...
336
307
 
337
308
  @typing.overload
338
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
309
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
339
310
  ...
340
311
 
341
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
312
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
342
313
  """
343
- Specifies that the step will success under all circumstances.
314
+ Enables loading / saving of models within a step.
344
315
 
345
- The decorator will create an optional artifact, specified by `var`, which
346
- contains the exception raised. You can use it to detect the presence
347
- of errors, indicating that all happy-path artifacts produced by the step
348
- are missing.
349
316
 
350
317
 
351
318
  Parameters
352
319
  ----------
353
- var : str, optional, default None
354
- Name of the artifact in which to store the caught exception.
355
- If not specified, the exception is not stored.
356
- print_exception : bool, default True
357
- Determines whether or not the exception is printed to
358
- stdout when caught.
320
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
321
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
322
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
323
+ - `current.checkpoint`
324
+ - `current.model`
325
+ - `current.huggingface_hub`
326
+
327
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
328
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
329
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
330
+
331
+ temp_dir_root : str, default: None
332
+ The root directory under which `current.model.loaded` will store loaded models
333
+ """
334
+ ...
335
+
336
+ @typing.overload
337
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
338
+ """
339
+ Specifies secrets to be retrieved and injected as environment variables prior to
340
+ the execution of a step.
341
+
342
+
343
+ Parameters
344
+ ----------
345
+ sources : List[Union[str, Dict[str, Any]]], default: []
346
+ List of secret specs, defining how the secrets are to be retrieved
347
+ """
348
+ ...
349
+
350
+ @typing.overload
351
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
352
+ ...
353
+
354
+ @typing.overload
355
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
356
+ ...
357
+
358
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
359
+ """
360
+ Specifies secrets to be retrieved and injected as environment variables prior to
361
+ the execution of a step.
362
+
363
+
364
+ Parameters
365
+ ----------
366
+ sources : List[Union[str, Dict[str, Any]]], default: []
367
+ List of secret specs, defining how the secrets are to be retrieved
359
368
  """
360
369
  ...
361
370
 
@@ -419,41 +428,223 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
419
428
  ...
420
429
 
421
430
  @typing.overload
422
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
431
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
423
432
  """
424
- Specifies the PyPI packages for the step.
425
-
426
- Information in this decorator will augment any
427
- attributes set in the `@pyi_base` flow-level decorator. Hence,
428
- you can use `@pypi_base` to set packages required by all
429
- steps and use `@pypi` to specify step-specific overrides.
430
-
431
-
432
- Parameters
433
- ----------
434
- packages : Dict[str, str], default: {}
435
- Packages to use for this step. The key is the name of the package
436
- and the value is the version to use.
437
- python : str, optional, default: None
438
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
439
- that the version used will correspond to the version of the Python interpreter used to start the run.
433
+ Internal decorator to support Fast bakery
440
434
  """
441
435
  ...
442
436
 
443
437
  @typing.overload
444
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
438
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
445
439
  ...
446
440
 
447
- @typing.overload
448
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
441
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
442
+ """
443
+ Internal decorator to support Fast bakery
444
+ """
449
445
  ...
450
446
 
451
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
447
+ @typing.overload
448
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
452
449
  """
453
- Specifies the PyPI packages for the step.
450
+ Specifies the Conda environment for the step.
454
451
 
455
452
  Information in this decorator will augment any
456
- attributes set in the `@pyi_base` flow-level decorator. Hence,
453
+ attributes set in the `@conda_base` flow-level decorator. Hence,
454
+ you can use `@conda_base` to set packages required by all
455
+ steps and use `@conda` to specify step-specific overrides.
456
+
457
+
458
+ Parameters
459
+ ----------
460
+ packages : Dict[str, str], default {}
461
+ Packages to use for this step. The key is the name of the package
462
+ and the value is the version to use.
463
+ libraries : Dict[str, str], default {}
464
+ Supported for backward compatibility. When used with packages, packages will take precedence.
465
+ python : str, optional, default None
466
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
467
+ that the version used will correspond to the version of the Python interpreter used to start the run.
468
+ disabled : bool, default False
469
+ If set to True, disables @conda.
470
+ """
471
+ ...
472
+
473
+ @typing.overload
474
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
475
+ ...
476
+
477
+ @typing.overload
478
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
479
+ ...
480
+
481
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
482
+ """
483
+ Specifies the Conda environment for the step.
484
+
485
+ Information in this decorator will augment any
486
+ attributes set in the `@conda_base` flow-level decorator. Hence,
487
+ you can use `@conda_base` to set packages required by all
488
+ steps and use `@conda` to specify step-specific overrides.
489
+
490
+
491
+ Parameters
492
+ ----------
493
+ packages : Dict[str, str], default {}
494
+ Packages to use for this step. The key is the name of the package
495
+ and the value is the version to use.
496
+ libraries : Dict[str, str], default {}
497
+ Supported for backward compatibility. When used with packages, packages will take precedence.
498
+ python : str, optional, default None
499
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
500
+ that the version used will correspond to the version of the Python interpreter used to start the run.
501
+ disabled : bool, default False
502
+ If set to True, disables @conda.
503
+ """
504
+ ...
505
+
506
+ @typing.overload
507
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
508
+ """
509
+ Specifies that the step will success under all circumstances.
510
+
511
+ The decorator will create an optional artifact, specified by `var`, which
512
+ contains the exception raised. You can use it to detect the presence
513
+ of errors, indicating that all happy-path artifacts produced by the step
514
+ are missing.
515
+
516
+
517
+ Parameters
518
+ ----------
519
+ var : str, optional, default None
520
+ Name of the artifact in which to store the caught exception.
521
+ If not specified, the exception is not stored.
522
+ print_exception : bool, default True
523
+ Determines whether or not the exception is printed to
524
+ stdout when caught.
525
+ """
526
+ ...
527
+
528
+ @typing.overload
529
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
530
+ ...
531
+
532
+ @typing.overload
533
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
534
+ ...
535
+
536
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
537
+ """
538
+ Specifies that the step will success under all circumstances.
539
+
540
+ The decorator will create an optional artifact, specified by `var`, which
541
+ contains the exception raised. You can use it to detect the presence
542
+ of errors, indicating that all happy-path artifacts produced by the step
543
+ are missing.
544
+
545
+
546
+ Parameters
547
+ ----------
548
+ var : str, optional, default None
549
+ Name of the artifact in which to store the caught exception.
550
+ If not specified, the exception is not stored.
551
+ print_exception : bool, default True
552
+ Determines whether or not the exception is printed to
553
+ stdout when caught.
554
+ """
555
+ ...
556
+
557
+ @typing.overload
558
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
559
+ """
560
+ Specifies the number of times the task corresponding
561
+ to a step needs to be retried.
562
+
563
+ This decorator is useful for handling transient errors, such as networking issues.
564
+ If your task contains operations that can't be retried safely, e.g. database updates,
565
+ it is advisable to annotate it with `@retry(times=0)`.
566
+
567
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
568
+ decorator will execute a no-op task after all retries have been exhausted,
569
+ ensuring that the flow execution can continue.
570
+
571
+
572
+ Parameters
573
+ ----------
574
+ times : int, default 3
575
+ Number of times to retry this task.
576
+ minutes_between_retries : int, default 2
577
+ Number of minutes between retries.
578
+ """
579
+ ...
580
+
581
+ @typing.overload
582
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
583
+ ...
584
+
585
+ @typing.overload
586
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
587
+ ...
588
+
589
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
590
+ """
591
+ Specifies the number of times the task corresponding
592
+ to a step needs to be retried.
593
+
594
+ This decorator is useful for handling transient errors, such as networking issues.
595
+ If your task contains operations that can't be retried safely, e.g. database updates,
596
+ it is advisable to annotate it with `@retry(times=0)`.
597
+
598
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
599
+ decorator will execute a no-op task after all retries have been exhausted,
600
+ ensuring that the flow execution can continue.
601
+
602
+
603
+ Parameters
604
+ ----------
605
+ times : int, default 3
606
+ Number of times to retry this task.
607
+ minutes_between_retries : int, default 2
608
+ Number of minutes between retries.
609
+ """
610
+ ...
611
+
612
+ @typing.overload
613
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
614
+ """
615
+ Specifies the PyPI packages for the step.
616
+
617
+ Information in this decorator will augment any
618
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
619
+ you can use `@pypi_base` to set packages required by all
620
+ steps and use `@pypi` to specify step-specific overrides.
621
+
622
+
623
+ Parameters
624
+ ----------
625
+ packages : Dict[str, str], default: {}
626
+ Packages to use for this step. The key is the name of the package
627
+ and the value is the version to use.
628
+ python : str, optional, default: None
629
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
630
+ that the version used will correspond to the version of the Python interpreter used to start the run.
631
+ """
632
+ ...
633
+
634
+ @typing.overload
635
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
636
+ ...
637
+
638
+ @typing.overload
639
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
640
+ ...
641
+
642
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
643
+ """
644
+ Specifies the PyPI packages for the step.
645
+
646
+ Information in this decorator will augment any
647
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
457
648
  you can use `@pypi_base` to set packages required by all
458
649
  steps and use `@pypi` to specify step-specific overrides.
459
650
 
@@ -469,36 +660,50 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
469
660
  """
470
661
  ...
471
662
 
472
- def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
663
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
664
+ """
665
+ Specifies that this step should execute on DGX cloud.
666
+
667
+
668
+ Parameters
669
+ ----------
670
+ gpu : int
671
+ Number of GPUs to use.
672
+ gpu_type : str
673
+ Type of Nvidia GPU to use.
674
+ """
675
+ ...
676
+
677
+ def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
473
678
  """
474
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
679
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
475
680
 
476
681
  User code call
477
682
  -----------
478
- @nim(
683
+ @ollama(
479
684
  models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
480
- backend='managed'
685
+ backend='local'
481
686
  )
482
687
 
483
688
  Valid backend options
484
689
  ---------------------
485
- - 'managed': Outerbounds selects a compute provider based on the model.
690
+ - 'local': Run as a separate process on the local task machine.
691
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
692
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
486
693
 
487
694
  Valid model options
488
695
  ----------------
489
- - 'meta/llama3-8b-instruct': 8B parameter model
490
- - 'meta/llama3-70b-instruct': 70B parameter model
491
- - any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
696
+ - 'llama3.2'
697
+ - 'llama3.3'
698
+ - any model here https://ollama.com/search
492
699
 
493
700
 
494
701
  Parameters
495
702
  ----------
496
- models: list[NIM]
497
- List of NIM containers running models in sidecars.
703
+ models: list[Ollama]
704
+ List of Ollama containers running models in sidecars.
498
705
  backend: str
499
- Compute provider to run the NIM container.
500
- queue_timeout : int
501
- Time to keep the job in NVCF's queue.
706
+ Determines where and how to run the Ollama process.
502
707
  """
503
708
  ...
504
709
 
@@ -582,198 +787,93 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
582
787
  ...
583
788
 
584
789
  @typing.overload
585
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
790
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
586
791
  """
587
- Internal decorator to support Fast bakery
792
+ Decorator prototype for all step decorators. This function gets specialized
793
+ and imported for all decorators types by _import_plugin_decorators().
588
794
  """
589
795
  ...
590
796
 
591
797
  @typing.overload
592
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
798
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
593
799
  ...
594
800
 
595
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
801
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
596
802
  """
597
- Internal decorator to support Fast bakery
803
+ Decorator prototype for all step decorators. This function gets specialized
804
+ and imported for all decorators types by _import_plugin_decorators().
598
805
  """
599
806
  ...
600
807
 
601
- @typing.overload
602
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
808
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
603
809
  """
604
- Specifies environment variables to be set prior to the execution of a step.
810
+ Specifies that this step is used to deploy an instance of the app.
811
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
605
812
 
606
813
 
607
814
  Parameters
608
815
  ----------
609
- vars : Dict[str, str], default {}
610
- Dictionary of environment variables to set.
816
+ app_port : int
817
+ Number of GPUs to use.
818
+ app_name : str
819
+ Name of the app to deploy.
611
820
  """
612
821
  ...
613
822
 
614
823
  @typing.overload
615
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
616
- ...
617
-
618
- @typing.overload
619
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
620
- ...
621
-
622
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
623
- """
624
- Specifies environment variables to be set prior to the execution of a step.
625
-
626
-
627
- Parameters
628
- ----------
629
- vars : Dict[str, str], default {}
630
- Dictionary of environment variables to set.
631
- """
632
- ...
633
-
634
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
635
- """
636
- Specifies that this step is used to deploy an instance of the app.
637
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
638
-
639
-
640
- Parameters
641
- ----------
642
- app_port : int
643
- Number of GPUs to use.
644
- app_name : str
645
- Name of the app to deploy.
646
- """
647
- ...
648
-
649
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
650
- """
651
- Specifies that this step should execute on DGX cloud.
652
-
653
-
654
- Parameters
655
- ----------
656
- gpu : int
657
- Number of GPUs to use.
658
- gpu_type : str
659
- Type of Nvidia GPU to use.
660
- """
661
- ...
662
-
663
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
664
- """
665
- Decorator that helps cache, version and store models/datasets from huggingface hub.
666
-
667
-
668
- Parameters
669
- ----------
670
- temp_dir_root : str, optional
671
- The root directory that will hold the temporary directory where objects will be downloaded.
672
-
673
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
674
- The list of repos (models/datasets) to load.
675
-
676
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
677
-
678
- - If repo (model/dataset) is not found in the datastore:
679
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
680
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
681
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
682
-
683
- - If repo is found in the datastore:
684
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
685
- """
686
- ...
687
-
688
- @typing.overload
689
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
824
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
690
825
  """
691
- Enables loading / saving of models within a step.
826
+ Enables checkpointing for a step.
692
827
 
693
828
 
694
829
 
695
830
  Parameters
696
831
  ----------
697
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
698
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
699
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
700
- - `current.checkpoint`
701
- - `current.model`
702
- - `current.huggingface_hub`
703
-
704
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
705
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
706
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
832
+ load_policy : str, default: "fresh"
833
+ The policy for loading the checkpoint. The following policies are supported:
834
+ - "eager": Loads the the latest available checkpoint within the namespace.
835
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
836
+ will be loaded at the start of the task.
837
+ - "none": Do not load any checkpoint
838
+ - "fresh": Loads the lastest checkpoint created within the running Task.
839
+ This mode helps loading checkpoints across various retry attempts of the same task.
840
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
841
+ created within the task will be loaded when the task is retries execution on failure.
707
842
 
708
843
  temp_dir_root : str, default: None
709
- The root directory under which `current.model.loaded` will store loaded models
844
+ The root directory under which `current.checkpoint.directory` will be created.
710
845
  """
711
846
  ...
712
847
 
713
848
  @typing.overload
714
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
849
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
715
850
  ...
716
851
 
717
852
  @typing.overload
718
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
853
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
719
854
  ...
720
855
 
721
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
856
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
722
857
  """
723
- Enables loading / saving of models within a step.
858
+ Enables checkpointing for a step.
724
859
 
725
860
 
726
861
 
727
862
  Parameters
728
863
  ----------
729
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
730
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
731
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
732
- - `current.checkpoint`
733
- - `current.model`
734
- - `current.huggingface_hub`
735
-
736
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
737
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
738
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
864
+ load_policy : str, default: "fresh"
865
+ The policy for loading the checkpoint. The following policies are supported:
866
+ - "eager": Loads the the latest available checkpoint within the namespace.
867
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
868
+ will be loaded at the start of the task.
869
+ - "none": Do not load any checkpoint
870
+ - "fresh": Loads the lastest checkpoint created within the running Task.
871
+ This mode helps loading checkpoints across various retry attempts of the same task.
872
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
873
+ created within the task will be loaded when the task is retries execution on failure.
739
874
 
740
875
  temp_dir_root : str, default: None
741
- The root directory under which `current.model.loaded` will store loaded models
742
- """
743
- ...
744
-
745
- @typing.overload
746
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
747
- """
748
- Specifies secrets to be retrieved and injected as environment variables prior to
749
- the execution of a step.
750
-
751
-
752
- Parameters
753
- ----------
754
- sources : List[Union[str, Dict[str, Any]]], default: []
755
- List of secret specs, defining how the secrets are to be retrieved
756
- """
757
- ...
758
-
759
- @typing.overload
760
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
761
- ...
762
-
763
- @typing.overload
764
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
765
- ...
766
-
767
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
768
- """
769
- Specifies secrets to be retrieved and injected as environment variables prior to
770
- the execution of a step.
771
-
772
-
773
- Parameters
774
- ----------
775
- sources : List[Union[str, Dict[str, Any]]], default: []
776
- List of secret specs, defining how the secrets are to be retrieved
876
+ The root directory under which `current.checkpoint.directory` will be created.
777
877
  """
778
878
  ...
779
879
 
@@ -850,148 +950,15 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
850
950
  Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
851
951
  Only applicable when @parallel is used.
852
952
  qos: str, default: Burstable
853
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
854
-
855
- security_context: Dict[str, Any], optional, default None
856
- Container security context. Applies to the task container. Allows the following keys:
857
- - privileged: bool, optional, default None
858
- - allow_privilege_escalation: bool, optional, default None
859
- - run_as_user: int, optional, default None
860
- - run_as_group: int, optional, default None
861
- - run_as_non_root: bool, optional, default None
862
- """
863
- ...
864
-
865
- @typing.overload
866
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
867
- """
868
- Decorator prototype for all step decorators. This function gets specialized
869
- and imported for all decorators types by _import_plugin_decorators().
870
- """
871
- ...
872
-
873
- @typing.overload
874
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
875
- ...
876
-
877
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
878
- """
879
- Decorator prototype for all step decorators. This function gets specialized
880
- and imported for all decorators types by _import_plugin_decorators().
881
- """
882
- ...
883
-
884
- @typing.overload
885
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
886
- """
887
- Specifies the number of times the task corresponding
888
- to a step needs to be retried.
889
-
890
- This decorator is useful for handling transient errors, such as networking issues.
891
- If your task contains operations that can't be retried safely, e.g. database updates,
892
- it is advisable to annotate it with `@retry(times=0)`.
893
-
894
- This can be used in conjunction with the `@catch` decorator. The `@catch`
895
- decorator will execute a no-op task after all retries have been exhausted,
896
- ensuring that the flow execution can continue.
897
-
898
-
899
- Parameters
900
- ----------
901
- times : int, default 3
902
- Number of times to retry this task.
903
- minutes_between_retries : int, default 2
904
- Number of minutes between retries.
905
- """
906
- ...
907
-
908
- @typing.overload
909
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
910
- ...
911
-
912
- @typing.overload
913
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
914
- ...
915
-
916
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
917
- """
918
- Specifies the number of times the task corresponding
919
- to a step needs to be retried.
920
-
921
- This decorator is useful for handling transient errors, such as networking issues.
922
- If your task contains operations that can't be retried safely, e.g. database updates,
923
- it is advisable to annotate it with `@retry(times=0)`.
924
-
925
- This can be used in conjunction with the `@catch` decorator. The `@catch`
926
- decorator will execute a no-op task after all retries have been exhausted,
927
- ensuring that the flow execution can continue.
928
-
929
-
930
- Parameters
931
- ----------
932
- times : int, default 3
933
- Number of times to retry this task.
934
- minutes_between_retries : int, default 2
935
- Number of minutes between retries.
936
- """
937
- ...
938
-
939
- @typing.overload
940
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
941
- """
942
- Specifies the Conda environment for the step.
943
-
944
- Information in this decorator will augment any
945
- attributes set in the `@conda_base` flow-level decorator. Hence,
946
- you can use `@conda_base` to set packages required by all
947
- steps and use `@conda` to specify step-specific overrides.
948
-
949
-
950
- Parameters
951
- ----------
952
- packages : Dict[str, str], default {}
953
- Packages to use for this step. The key is the name of the package
954
- and the value is the version to use.
955
- libraries : Dict[str, str], default {}
956
- Supported for backward compatibility. When used with packages, packages will take precedence.
957
- python : str, optional, default None
958
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
959
- that the version used will correspond to the version of the Python interpreter used to start the run.
960
- disabled : bool, default False
961
- If set to True, disables @conda.
962
- """
963
- ...
964
-
965
- @typing.overload
966
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
967
- ...
968
-
969
- @typing.overload
970
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
971
- ...
972
-
973
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
974
- """
975
- Specifies the Conda environment for the step.
976
-
977
- Information in this decorator will augment any
978
- attributes set in the `@conda_base` flow-level decorator. Hence,
979
- you can use `@conda_base` to set packages required by all
980
- steps and use `@conda` to specify step-specific overrides.
981
-
953
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
982
954
 
983
- Parameters
984
- ----------
985
- packages : Dict[str, str], default {}
986
- Packages to use for this step. The key is the name of the package
987
- and the value is the version to use.
988
- libraries : Dict[str, str], default {}
989
- Supported for backward compatibility. When used with packages, packages will take precedence.
990
- python : str, optional, default None
991
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
992
- that the version used will correspond to the version of the Python interpreter used to start the run.
993
- disabled : bool, default False
994
- If set to True, disables @conda.
955
+ security_context: Dict[str, Any], optional, default None
956
+ Container security context. Applies to the task container. Allows the following keys:
957
+ - privileged: bool, optional, default None
958
+ - allow_privilege_escalation: bool, optional, default None
959
+ - run_as_user: int, optional, default None
960
+ - run_as_group: int, optional, default None
961
+ - run_as_non_root: bool, optional, default None
995
962
  """
996
963
  ...
997
964
 
@@ -1088,6 +1055,120 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1088
1055
  """
1089
1056
  ...
1090
1057
 
1058
+ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1059
+ """
1060
+ Allows setting external datastores to save data for the
1061
+ `@checkpoint`/`@model`/`@huggingface_hub` decorators.
1062
+
1063
+ This decorator is useful when users wish to save data to a different datastore
1064
+ than what is configured in Metaflow. This can be for variety of reasons:
1065
+
1066
+ 1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
1067
+ 2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
1068
+ - Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
1069
+ 3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
1070
+ - Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
1071
+
1072
+ Usage:
1073
+ ----------
1074
+
1075
+ - Using a custom IAM role to access the datastore.
1076
+
1077
+ ```python
1078
+ @with_artifact_store(
1079
+ type="s3",
1080
+ config=lambda: {
1081
+ "root": "s3://my-bucket-foo/path/to/root",
1082
+ "role_arn": ROLE,
1083
+ },
1084
+ )
1085
+ class MyFlow(FlowSpec):
1086
+
1087
+ @checkpoint
1088
+ @step
1089
+ def start(self):
1090
+ with open("my_file.txt", "w") as f:
1091
+ f.write("Hello, World!")
1092
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1093
+ self.next(self.end)
1094
+
1095
+ ```
1096
+
1097
+ - Using credentials to access the s3-compatible datastore.
1098
+
1099
+ ```python
1100
+ @with_artifact_store(
1101
+ type="s3",
1102
+ config=lambda: {
1103
+ "root": "s3://my-bucket-foo/path/to/root",
1104
+ "client_params": {
1105
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1106
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1107
+ },
1108
+ },
1109
+ )
1110
+ class MyFlow(FlowSpec):
1111
+
1112
+ @checkpoint
1113
+ @step
1114
+ def start(self):
1115
+ with open("my_file.txt", "w") as f:
1116
+ f.write("Hello, World!")
1117
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1118
+ self.next(self.end)
1119
+
1120
+ ```
1121
+
1122
+ - Accessing objects stored in external datastores after task execution.
1123
+
1124
+ ```python
1125
+ run = Run("CheckpointsTestsFlow/8992")
1126
+ with artifact_store_from(run=run, config={
1127
+ "client_params": {
1128
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1129
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1130
+ },
1131
+ }):
1132
+ with Checkpoint() as cp:
1133
+ latest = cp.list(
1134
+ task=run["start"].task
1135
+ )[0]
1136
+ print(latest)
1137
+ cp.load(
1138
+ latest,
1139
+ "test-checkpoints"
1140
+ )
1141
+
1142
+ task = Task("TorchTuneFlow/8484/train/53673")
1143
+ with artifact_store_from(run=run, config={
1144
+ "client_params": {
1145
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1146
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1147
+ },
1148
+ }):
1149
+ load_model(
1150
+ task.data.model_ref,
1151
+ "test-models"
1152
+ )
1153
+ ```
1154
+ Parameters:
1155
+ ----------
1156
+
1157
+ type: str
1158
+ The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1159
+
1160
+ config: dict or Callable
1161
+ Dictionary of configuration options for the datastore. The following keys are required:
1162
+ - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1163
+ - example: 's3://bucket-name/path/to/root'
1164
+ - example: 'gs://bucket-name/path/to/root'
1165
+ - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1166
+ - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1167
+ - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1168
+ - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1169
+ """
1170
+ ...
1171
+
1091
1172
  def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1092
1173
  """
1093
1174
  The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
@@ -1132,53 +1213,43 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1132
1213
  ...
1133
1214
 
1134
1215
  @typing.overload
1135
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1216
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1136
1217
  """
1137
- Specifies the times when the flow should be run when running on a
1138
- production scheduler.
1218
+ Specifies the PyPI packages for all steps of the flow.
1139
1219
 
1220
+ Use `@pypi_base` to set common packages required by all
1221
+ steps and use `@pypi` to specify step-specific overrides.
1140
1222
 
1141
1223
  Parameters
1142
1224
  ----------
1143
- hourly : bool, default False
1144
- Run the workflow hourly.
1145
- daily : bool, default True
1146
- Run the workflow daily.
1147
- weekly : bool, default False
1148
- Run the workflow weekly.
1149
- cron : str, optional, default None
1150
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1151
- specified by this expression.
1152
- timezone : str, optional, default None
1153
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1154
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1225
+ packages : Dict[str, str], default: {}
1226
+ Packages to use for this flow. The key is the name of the package
1227
+ and the value is the version to use.
1228
+ python : str, optional, default: None
1229
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1230
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1155
1231
  """
1156
1232
  ...
1157
1233
 
1158
1234
  @typing.overload
1159
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1235
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1160
1236
  ...
1161
1237
 
1162
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1238
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1163
1239
  """
1164
- Specifies the times when the flow should be run when running on a
1165
- production scheduler.
1240
+ Specifies the PyPI packages for all steps of the flow.
1166
1241
 
1242
+ Use `@pypi_base` to set common packages required by all
1243
+ steps and use `@pypi` to specify step-specific overrides.
1167
1244
 
1168
1245
  Parameters
1169
1246
  ----------
1170
- hourly : bool, default False
1171
- Run the workflow hourly.
1172
- daily : bool, default True
1173
- Run the workflow daily.
1174
- weekly : bool, default False
1175
- Run the workflow weekly.
1176
- cron : str, optional, default None
1177
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1178
- specified by this expression.
1179
- timezone : str, optional, default None
1180
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1181
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1247
+ packages : Dict[str, str], default: {}
1248
+ Packages to use for this flow. The key is the name of the package
1249
+ and the value is the version to use.
1250
+ python : str, optional, default: None
1251
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1252
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1182
1253
  """
1183
1254
  ...
1184
1255
 
@@ -1193,86 +1264,35 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1193
1264
 
1194
1265
  Parameters
1195
1266
  ----------
1196
- timeout : int
1197
- Time, in seconds before the task times out and fails. (Default: 3600)
1198
- poke_interval : int
1199
- Time in seconds that the job should wait in between each try. (Default: 60)
1200
- mode : str
1201
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1202
- exponential_backoff : bool
1203
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1204
- pool : str
1205
- the slot pool this task should run in,
1206
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1207
- soft_fail : bool
1208
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1209
- name : str
1210
- Name of the sensor on Airflow
1211
- description : str
1212
- Description of sensor in the Airflow UI
1213
- bucket_key : Union[str, List[str]]
1214
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1215
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1216
- bucket_name : str
1217
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1218
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1219
- wildcard_match : bool
1220
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1221
- aws_conn_id : str
1222
- a reference to the s3 connection on Airflow. (Default: None)
1223
- verify : bool
1224
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1225
- """
1226
- ...
1227
-
1228
- @typing.overload
1229
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1230
- """
1231
- Specifies the Conda environment for all steps of the flow.
1232
-
1233
- Use `@conda_base` to set common libraries required by all
1234
- steps and use `@conda` to specify step-specific additions.
1235
-
1236
-
1237
- Parameters
1238
- ----------
1239
- packages : Dict[str, str], default {}
1240
- Packages to use for this flow. The key is the name of the package
1241
- and the value is the version to use.
1242
- libraries : Dict[str, str], default {}
1243
- Supported for backward compatibility. When used with packages, packages will take precedence.
1244
- python : str, optional, default None
1245
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1246
- that the version used will correspond to the version of the Python interpreter used to start the run.
1247
- disabled : bool, default False
1248
- If set to True, disables Conda.
1249
- """
1250
- ...
1251
-
1252
- @typing.overload
1253
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1254
- ...
1255
-
1256
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1257
- """
1258
- Specifies the Conda environment for all steps of the flow.
1259
-
1260
- Use `@conda_base` to set common libraries required by all
1261
- steps and use `@conda` to specify step-specific additions.
1262
-
1263
-
1264
- Parameters
1265
- ----------
1266
- packages : Dict[str, str], default {}
1267
- Packages to use for this flow. The key is the name of the package
1268
- and the value is the version to use.
1269
- libraries : Dict[str, str], default {}
1270
- Supported for backward compatibility. When used with packages, packages will take precedence.
1271
- python : str, optional, default None
1272
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1273
- that the version used will correspond to the version of the Python interpreter used to start the run.
1274
- disabled : bool, default False
1275
- If set to True, disables Conda.
1267
+ timeout : int
1268
+ Time, in seconds before the task times out and fails. (Default: 3600)
1269
+ poke_interval : int
1270
+ Time in seconds that the job should wait in between each try. (Default: 60)
1271
+ mode : str
1272
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1273
+ exponential_backoff : bool
1274
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1275
+ pool : str
1276
+ the slot pool this task should run in,
1277
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1278
+ soft_fail : bool
1279
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1280
+ name : str
1281
+ Name of the sensor on Airflow
1282
+ description : str
1283
+ Description of sensor in the Airflow UI
1284
+ bucket_key : Union[str, List[str]]
1285
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1286
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1287
+ bucket_name : str
1288
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1289
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1290
+ wildcard_match : bool
1291
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1292
+ aws_conn_id : str
1293
+ a reference to the s3 connection on Airflow. (Default: None)
1294
+ verify : bool
1295
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1276
1296
  """
1277
1297
  ...
1278
1298
 
@@ -1377,158 +1397,54 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1377
1397
  """
1378
1398
  ...
1379
1399
 
1380
- def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1381
- """
1382
- Allows setting external datastores to save data for the
1383
- `@checkpoint`/`@model`/`@huggingface_hub` decorators.
1384
-
1385
- This decorator is useful when users wish to save data to a different datastore
1386
- than what is configured in Metaflow. This can be for variety of reasons:
1387
-
1388
- 1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
1389
- 2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
1390
- - Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
1391
- 3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
1392
- - Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
1393
-
1394
- Usage:
1395
- ----------
1396
-
1397
- - Using a custom IAM role to access the datastore.
1398
-
1399
- ```python
1400
- @with_artifact_store(
1401
- type="s3",
1402
- config=lambda: {
1403
- "root": "s3://my-bucket-foo/path/to/root",
1404
- "role_arn": ROLE,
1405
- },
1406
- )
1407
- class MyFlow(FlowSpec):
1408
-
1409
- @checkpoint
1410
- @step
1411
- def start(self):
1412
- with open("my_file.txt", "w") as f:
1413
- f.write("Hello, World!")
1414
- self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1415
- self.next(self.end)
1416
-
1417
- ```
1418
-
1419
- - Using credentials to access the s3-compatible datastore.
1420
-
1421
- ```python
1422
- @with_artifact_store(
1423
- type="s3",
1424
- config=lambda: {
1425
- "root": "s3://my-bucket-foo/path/to/root",
1426
- "client_params": {
1427
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1428
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1429
- },
1430
- },
1431
- )
1432
- class MyFlow(FlowSpec):
1433
-
1434
- @checkpoint
1435
- @step
1436
- def start(self):
1437
- with open("my_file.txt", "w") as f:
1438
- f.write("Hello, World!")
1439
- self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1440
- self.next(self.end)
1441
-
1442
- ```
1443
-
1444
- - Accessing objects stored in external datastores after task execution.
1445
-
1446
- ```python
1447
- run = Run("CheckpointsTestsFlow/8992")
1448
- with artifact_store_from(run=run, config={
1449
- "client_params": {
1450
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1451
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1452
- },
1453
- }):
1454
- with Checkpoint() as cp:
1455
- latest = cp.list(
1456
- task=run["start"].task
1457
- )[0]
1458
- print(latest)
1459
- cp.load(
1460
- latest,
1461
- "test-checkpoints"
1462
- )
1463
-
1464
- task = Task("TorchTuneFlow/8484/train/53673")
1465
- with artifact_store_from(run=run, config={
1466
- "client_params": {
1467
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1468
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1469
- },
1470
- }):
1471
- load_model(
1472
- task.data.model_ref,
1473
- "test-models"
1474
- )
1475
- ```
1476
- Parameters:
1477
- ----------
1478
-
1479
- type: str
1480
- The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1481
-
1482
- config: dict or Callable
1483
- Dictionary of configuration options for the datastore. The following keys are required:
1484
- - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1485
- - example: 's3://bucket-name/path/to/root'
1486
- - example: 'gs://bucket-name/path/to/root'
1487
- - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1488
- - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1489
- - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1490
- - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1491
- """
1492
- ...
1493
-
1494
1400
  @typing.overload
1495
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1401
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1496
1402
  """
1497
- Specifies the PyPI packages for all steps of the flow.
1403
+ Specifies the times when the flow should be run when running on a
1404
+ production scheduler.
1498
1405
 
1499
- Use `@pypi_base` to set common packages required by all
1500
- steps and use `@pypi` to specify step-specific overrides.
1501
1406
 
1502
1407
  Parameters
1503
1408
  ----------
1504
- packages : Dict[str, str], default: {}
1505
- Packages to use for this flow. The key is the name of the package
1506
- and the value is the version to use.
1507
- python : str, optional, default: None
1508
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1509
- that the version used will correspond to the version of the Python interpreter used to start the run.
1409
+ hourly : bool, default False
1410
+ Run the workflow hourly.
1411
+ daily : bool, default True
1412
+ Run the workflow daily.
1413
+ weekly : bool, default False
1414
+ Run the workflow weekly.
1415
+ cron : str, optional, default None
1416
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1417
+ specified by this expression.
1418
+ timezone : str, optional, default None
1419
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1420
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1510
1421
  """
1511
1422
  ...
1512
1423
 
1513
1424
  @typing.overload
1514
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1425
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1515
1426
  ...
1516
1427
 
1517
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1428
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1518
1429
  """
1519
- Specifies the PyPI packages for all steps of the flow.
1430
+ Specifies the times when the flow should be run when running on a
1431
+ production scheduler.
1520
1432
 
1521
- Use `@pypi_base` to set common packages required by all
1522
- steps and use `@pypi` to specify step-specific overrides.
1523
1433
 
1524
1434
  Parameters
1525
1435
  ----------
1526
- packages : Dict[str, str], default: {}
1527
- Packages to use for this flow. The key is the name of the package
1528
- and the value is the version to use.
1529
- python : str, optional, default: None
1530
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1531
- that the version used will correspond to the version of the Python interpreter used to start the run.
1436
+ hourly : bool, default False
1437
+ Run the workflow hourly.
1438
+ daily : bool, default True
1439
+ Run the workflow daily.
1440
+ weekly : bool, default False
1441
+ Run the workflow weekly.
1442
+ cron : str, optional, default None
1443
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1444
+ specified by this expression.
1445
+ timezone : str, optional, default None
1446
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1447
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1532
1448
  """
1533
1449
  ...
1534
1450
 
@@ -1567,5 +1483,56 @@ def project(*, name: str, branch: typing.Optional[str] = None, production: bool
1567
1483
  """
1568
1484
  ...
1569
1485
 
1486
+ @typing.overload
1487
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1488
+ """
1489
+ Specifies the Conda environment for all steps of the flow.
1490
+
1491
+ Use `@conda_base` to set common libraries required by all
1492
+ steps and use `@conda` to specify step-specific additions.
1493
+
1494
+
1495
+ Parameters
1496
+ ----------
1497
+ packages : Dict[str, str], default {}
1498
+ Packages to use for this flow. The key is the name of the package
1499
+ and the value is the version to use.
1500
+ libraries : Dict[str, str], default {}
1501
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1502
+ python : str, optional, default None
1503
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1504
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1505
+ disabled : bool, default False
1506
+ If set to True, disables Conda.
1507
+ """
1508
+ ...
1509
+
1510
+ @typing.overload
1511
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1512
+ ...
1513
+
1514
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1515
+ """
1516
+ Specifies the Conda environment for all steps of the flow.
1517
+
1518
+ Use `@conda_base` to set common libraries required by all
1519
+ steps and use `@conda` to specify step-specific additions.
1520
+
1521
+
1522
+ Parameters
1523
+ ----------
1524
+ packages : Dict[str, str], default {}
1525
+ Packages to use for this flow. The key is the name of the package
1526
+ and the value is the version to use.
1527
+ libraries : Dict[str, str], default {}
1528
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1529
+ python : str, optional, default None
1530
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1531
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1532
+ disabled : bool, default False
1533
+ If set to True, disables Conda.
1534
+ """
1535
+ ...
1536
+
1570
1537
  pkg_name: str
1571
1538