ob-metaflow-stubs 6.0.3.170__py2.py3-none-any.whl → 6.0.3.171__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (208) hide show
  1. metaflow-stubs/__init__.pyi +596 -592
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +1 -1
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +48 -48
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +1 -1
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +4 -4
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +1 -1
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +12 -0
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +30 -0
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +15 -7
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  77. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  78. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  79. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  81. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
  82. metaflow-stubs/multicore_utils.pyi +1 -1
  83. metaflow-stubs/parameters.pyi +3 -3
  84. metaflow-stubs/plugins/__init__.pyi +9 -9
  85. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  86. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  87. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  88. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  89. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  90. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  91. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  92. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  93. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  94. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  95. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  96. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  97. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  98. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  99. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  100. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  101. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  102. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  103. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  104. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  105. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  106. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  107. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  108. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  109. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  110. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  111. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  112. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  113. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  114. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  115. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  116. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  117. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  118. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  119. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  120. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  121. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  122. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  123. metaflow-stubs/plugins/cards/card_client.pyi +1 -1
  124. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  125. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  126. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  127. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  128. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  129. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  130. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  131. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  132. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  133. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  134. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  135. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  136. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  137. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  138. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  139. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  140. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  141. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  142. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  143. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  144. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  145. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  146. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  147. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  148. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  149. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  150. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  151. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  152. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  153. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  154. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  155. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  156. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  157. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  158. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  159. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  160. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  161. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  162. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  163. metaflow-stubs/plugins/ollama/__init__.pyi +17 -9
  164. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  165. metaflow-stubs/plugins/perimeters.pyi +1 -1
  166. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  167. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  168. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  169. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  170. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  171. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  172. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  173. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  174. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  175. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  176. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  177. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  178. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  179. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  180. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  181. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
  182. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  183. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  184. metaflow-stubs/plugins/uv/uv_environment.pyi +1 -1
  185. metaflow-stubs/profilers/__init__.pyi +1 -1
  186. metaflow-stubs/pylint_wrapper.pyi +1 -1
  187. metaflow-stubs/runner/__init__.pyi +1 -1
  188. metaflow-stubs/runner/deployer.pyi +6 -6
  189. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  190. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  191. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  192. metaflow-stubs/runner/nbrun.pyi +1 -1
  193. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  194. metaflow-stubs/runner/utils.pyi +1 -1
  195. metaflow-stubs/system/__init__.pyi +1 -1
  196. metaflow-stubs/system/system_logger.pyi +1 -1
  197. metaflow-stubs/system/system_monitor.pyi +1 -1
  198. metaflow-stubs/tagging_util.pyi +1 -1
  199. metaflow-stubs/tuple_util.pyi +1 -1
  200. metaflow-stubs/user_configs/__init__.pyi +1 -1
  201. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  202. metaflow-stubs/user_configs/config_options.pyi +3 -3
  203. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  204. {ob_metaflow_stubs-6.0.3.170.dist-info → ob_metaflow_stubs-6.0.3.171.dist-info}/METADATA +1 -1
  205. ob_metaflow_stubs-6.0.3.171.dist-info/RECORD +208 -0
  206. ob_metaflow_stubs-6.0.3.170.dist-info/RECORD +0 -206
  207. {ob_metaflow_stubs-6.0.3.170.dist-info → ob_metaflow_stubs-6.0.3.171.dist-info}/WHEEL +0 -0
  208. {ob_metaflow_stubs-6.0.3.170.dist-info → ob_metaflow_stubs-6.0.3.171.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.15.14.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-05-22T07:40:40.319392 #
4
+ # Generated on 2025-05-27T00:40:42.234892 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -45,8 +45,8 @@ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package imp
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
47
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
48
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
49
48
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
49
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
52
52
  from .client.core import get_namespace as get_namespace
@@ -153,218 +153,172 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
153
153
  """
154
154
  ...
155
155
 
156
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
157
- """
158
- Decorator that helps cache, version and store models/datasets from huggingface hub.
159
-
160
-
161
- Parameters
162
- ----------
163
- temp_dir_root : str, optional
164
- The root directory that will hold the temporary directory where objects will be downloaded.
165
-
166
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
167
- The list of repos (models/datasets) to load.
168
-
169
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
170
-
171
- - If repo (model/dataset) is not found in the datastore:
172
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
173
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
174
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
175
-
176
- - If repo is found in the datastore:
177
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
178
- """
179
- ...
180
-
181
- @typing.overload
182
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
183
- """
184
- Specifies environment variables to be set prior to the execution of a step.
185
-
186
-
187
- Parameters
188
- ----------
189
- vars : Dict[str, str], default {}
190
- Dictionary of environment variables to set.
191
- """
192
- ...
193
-
194
156
  @typing.overload
195
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
196
- ...
197
-
198
- @typing.overload
199
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
200
- ...
201
-
202
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
157
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
203
158
  """
204
- Specifies environment variables to be set prior to the execution of a step.
205
-
159
+ Specifies the number of times the task corresponding
160
+ to a step needs to be retried.
206
161
 
207
- Parameters
208
- ----------
209
- vars : Dict[str, str], default {}
210
- Dictionary of environment variables to set.
211
- """
212
- ...
213
-
214
- @typing.overload
215
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
216
- """
217
- Creates a human-readable report, a Metaflow Card, after this step completes.
162
+ This decorator is useful for handling transient errors, such as networking issues.
163
+ If your task contains operations that can't be retried safely, e.g. database updates,
164
+ it is advisable to annotate it with `@retry(times=0)`.
218
165
 
219
- Note that you may add multiple `@card` decorators in a step with different parameters.
166
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
167
+ decorator will execute a no-op task after all retries have been exhausted,
168
+ ensuring that the flow execution can continue.
220
169
 
221
170
 
222
171
  Parameters
223
172
  ----------
224
- type : str, default 'default'
225
- Card type.
226
- id : str, optional, default None
227
- If multiple cards are present, use this id to identify this card.
228
- options : Dict[str, Any], default {}
229
- Options passed to the card. The contents depend on the card type.
230
- timeout : int, default 45
231
- Interrupt reporting if it takes more than this many seconds.
173
+ times : int, default 3
174
+ Number of times to retry this task.
175
+ minutes_between_retries : int, default 2
176
+ Number of minutes between retries.
232
177
  """
233
178
  ...
234
179
 
235
180
  @typing.overload
236
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
181
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
237
182
  ...
238
183
 
239
184
  @typing.overload
240
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
185
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
241
186
  ...
242
187
 
243
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
188
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
244
189
  """
245
- Creates a human-readable report, a Metaflow Card, after this step completes.
246
-
247
- Note that you may add multiple `@card` decorators in a step with different parameters.
190
+ Specifies the number of times the task corresponding
191
+ to a step needs to be retried.
248
192
 
193
+ This decorator is useful for handling transient errors, such as networking issues.
194
+ If your task contains operations that can't be retried safely, e.g. database updates,
195
+ it is advisable to annotate it with `@retry(times=0)`.
249
196
 
250
- Parameters
251
- ----------
252
- type : str, default 'default'
253
- Card type.
254
- id : str, optional, default None
255
- If multiple cards are present, use this id to identify this card.
256
- options : Dict[str, Any], default {}
257
- Options passed to the card. The contents depend on the card type.
258
- timeout : int, default 45
259
- Interrupt reporting if it takes more than this many seconds.
260
- """
261
- ...
262
-
263
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
264
- """
265
- Specifies that this step should execute on DGX cloud.
197
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
198
+ decorator will execute a no-op task after all retries have been exhausted,
199
+ ensuring that the flow execution can continue.
266
200
 
267
201
 
268
202
  Parameters
269
203
  ----------
270
- gpu : int
271
- Number of GPUs to use.
272
- gpu_type : str
273
- Type of Nvidia GPU to use.
274
- queue_timeout : int
275
- Time to keep the job in NVCF's queue.
204
+ times : int, default 3
205
+ Number of times to retry this task.
206
+ minutes_between_retries : int, default 2
207
+ Number of minutes between retries.
276
208
  """
277
209
  ...
278
210
 
279
211
  @typing.overload
280
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
212
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
281
213
  """
282
- Enables loading / saving of models within a step.
283
-
214
+ Specifies secrets to be retrieved and injected as environment variables prior to
215
+ the execution of a step.
284
216
 
285
217
 
286
218
  Parameters
287
219
  ----------
288
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
289
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
290
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
291
- - `current.checkpoint`
292
- - `current.model`
293
- - `current.huggingface_hub`
294
-
295
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
296
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
297
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
298
-
299
- temp_dir_root : str, default: None
300
- The root directory under which `current.model.loaded` will store loaded models
220
+ sources : List[Union[str, Dict[str, Any]]], default: []
221
+ List of secret specs, defining how the secrets are to be retrieved
301
222
  """
302
223
  ...
303
224
 
304
225
  @typing.overload
305
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
226
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
306
227
  ...
307
228
 
308
229
  @typing.overload
309
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
230
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
310
231
  ...
311
232
 
312
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
233
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
313
234
  """
314
- Enables loading / saving of models within a step.
315
-
235
+ Specifies secrets to be retrieved and injected as environment variables prior to
236
+ the execution of a step.
316
237
 
317
238
 
318
239
  Parameters
319
240
  ----------
320
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
321
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
322
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
323
- - `current.checkpoint`
324
- - `current.model`
325
- - `current.huggingface_hub`
326
-
327
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
328
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
329
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
330
-
331
- temp_dir_root : str, default: None
332
- The root directory under which `current.model.loaded` will store loaded models
241
+ sources : List[Union[str, Dict[str, Any]]], default: []
242
+ List of secret specs, defining how the secrets are to be retrieved
333
243
  """
334
244
  ...
335
245
 
336
246
  @typing.overload
337
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
247
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
338
248
  """
339
- Specifies secrets to be retrieved and injected as environment variables prior to
340
- the execution of a step.
249
+ Specifies the resources needed when executing this step.
250
+
251
+ Use `@resources` to specify the resource requirements
252
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
253
+
254
+ You can choose the compute layer on the command line by executing e.g.
255
+ ```
256
+ python myflow.py run --with batch
257
+ ```
258
+ or
259
+ ```
260
+ python myflow.py run --with kubernetes
261
+ ```
262
+ which executes the flow on the desired system using the
263
+ requirements specified in `@resources`.
341
264
 
342
265
 
343
266
  Parameters
344
267
  ----------
345
- sources : List[Union[str, Dict[str, Any]]], default: []
346
- List of secret specs, defining how the secrets are to be retrieved
268
+ cpu : int, default 1
269
+ Number of CPUs required for this step.
270
+ gpu : int, optional, default None
271
+ Number of GPUs required for this step.
272
+ disk : int, optional, default None
273
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
274
+ memory : int, default 4096
275
+ Memory size (in MB) required for this step.
276
+ shared_memory : int, optional, default None
277
+ The value for the size (in MiB) of the /dev/shm volume for this step.
278
+ This parameter maps to the `--shm-size` option in Docker.
347
279
  """
348
280
  ...
349
281
 
350
282
  @typing.overload
351
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
283
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
352
284
  ...
353
285
 
354
286
  @typing.overload
355
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
287
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
356
288
  ...
357
289
 
358
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
290
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
359
291
  """
360
- Specifies secrets to be retrieved and injected as environment variables prior to
361
- the execution of a step.
292
+ Specifies the resources needed when executing this step.
293
+
294
+ Use `@resources` to specify the resource requirements
295
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
296
+
297
+ You can choose the compute layer on the command line by executing e.g.
298
+ ```
299
+ python myflow.py run --with batch
300
+ ```
301
+ or
302
+ ```
303
+ python myflow.py run --with kubernetes
304
+ ```
305
+ which executes the flow on the desired system using the
306
+ requirements specified in `@resources`.
362
307
 
363
308
 
364
309
  Parameters
365
310
  ----------
366
- sources : List[Union[str, Dict[str, Any]]], default: []
367
- List of secret specs, defining how the secrets are to be retrieved
311
+ cpu : int, default 1
312
+ Number of CPUs required for this step.
313
+ gpu : int, optional, default None
314
+ Number of GPUs required for this step.
315
+ disk : int, optional, default None
316
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
317
+ memory : int, default 4096
318
+ Memory size (in MB) required for this step.
319
+ shared_memory : int, optional, default None
320
+ The value for the size (in MiB) of the /dev/shm volume for this step.
321
+ This parameter maps to the `--shm-size` option in Docker.
368
322
  """
369
323
  ...
370
324
 
@@ -427,79 +381,164 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
427
381
  """
428
382
  ...
429
383
 
430
- @typing.overload
431
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
432
- """
433
- Internal decorator to support Fast bakery
434
- """
435
- ...
436
-
437
- @typing.overload
438
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
439
- ...
440
-
441
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
442
- """
443
- Internal decorator to support Fast bakery
384
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
444
385
  """
445
- ...
386
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
387
+
388
+
389
+ Parameters
390
+ ----------
391
+ temp_dir_root : str, optional
392
+ The root directory that will hold the temporary directory where objects will be downloaded.
393
+
394
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
395
+ The list of repos (models/datasets) to load.
396
+
397
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
398
+
399
+ - If repo (model/dataset) is not found in the datastore:
400
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
401
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
402
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
403
+
404
+ - If repo is found in the datastore:
405
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
406
+ """
407
+ ...
408
+
409
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
410
+ """
411
+ Specifies that this step is used to deploy an instance of the app.
412
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
413
+
414
+
415
+ Parameters
416
+ ----------
417
+ app_port : int
418
+ Number of GPUs to use.
419
+ app_name : str
420
+ Name of the app to deploy.
421
+ """
422
+ ...
423
+
424
+ def ollama(*, models: list, backend: str, force_pull: bool, skip_push_check: bool, debug: bool) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
425
+ """
426
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
427
+
428
+ User code call
429
+ --------------
430
+ @ollama(
431
+ models=[...],
432
+ ...
433
+ )
434
+
435
+ Valid backend options
436
+ ---------------------
437
+ - 'local': Run as a separate process on the local task machine.
438
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
439
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
440
+
441
+ Valid model options
442
+ -------------------
443
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
444
+
445
+
446
+ Parameters
447
+ ----------
448
+ models: list[str]
449
+ List of Ollama containers running models in sidecars.
450
+ backend: str
451
+ Determines where and how to run the Ollama process.
452
+ force_pull: bool
453
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
454
+ skip_push_check: bool
455
+ Whether to skip the check that populates/overwrites remote cache on terminating an ollama model.
456
+ debug: bool
457
+ Whether to turn on verbose debugging logs.
458
+ """
459
+ ...
446
460
 
447
461
  @typing.overload
448
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
462
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
449
463
  """
450
- Specifies the Conda environment for the step.
464
+ Specifies environment variables to be set prior to the execution of a step.
465
+
466
+
467
+ Parameters
468
+ ----------
469
+ vars : Dict[str, str], default {}
470
+ Dictionary of environment variables to set.
471
+ """
472
+ ...
473
+
474
+ @typing.overload
475
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
476
+ ...
477
+
478
+ @typing.overload
479
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
480
+ ...
481
+
482
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
483
+ """
484
+ Specifies environment variables to be set prior to the execution of a step.
485
+
486
+
487
+ Parameters
488
+ ----------
489
+ vars : Dict[str, str], default {}
490
+ Dictionary of environment variables to set.
491
+ """
492
+ ...
493
+
494
+ @typing.overload
495
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
496
+ """
497
+ Specifies the PyPI packages for the step.
451
498
 
452
499
  Information in this decorator will augment any
453
- attributes set in the `@conda_base` flow-level decorator. Hence,
454
- you can use `@conda_base` to set packages required by all
455
- steps and use `@conda` to specify step-specific overrides.
500
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
501
+ you can use `@pypi_base` to set packages required by all
502
+ steps and use `@pypi` to specify step-specific overrides.
456
503
 
457
504
 
458
505
  Parameters
459
506
  ----------
460
- packages : Dict[str, str], default {}
507
+ packages : Dict[str, str], default: {}
461
508
  Packages to use for this step. The key is the name of the package
462
509
  and the value is the version to use.
463
- libraries : Dict[str, str], default {}
464
- Supported for backward compatibility. When used with packages, packages will take precedence.
465
- python : str, optional, default None
510
+ python : str, optional, default: None
466
511
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
467
512
  that the version used will correspond to the version of the Python interpreter used to start the run.
468
- disabled : bool, default False
469
- If set to True, disables @conda.
470
513
  """
471
514
  ...
472
515
 
473
516
  @typing.overload
474
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
517
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
475
518
  ...
476
519
 
477
520
  @typing.overload
478
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
521
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
479
522
  ...
480
523
 
481
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
524
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
482
525
  """
483
- Specifies the Conda environment for the step.
526
+ Specifies the PyPI packages for the step.
484
527
 
485
528
  Information in this decorator will augment any
486
- attributes set in the `@conda_base` flow-level decorator. Hence,
487
- you can use `@conda_base` to set packages required by all
488
- steps and use `@conda` to specify step-specific overrides.
529
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
530
+ you can use `@pypi_base` to set packages required by all
531
+ steps and use `@pypi` to specify step-specific overrides.
489
532
 
490
533
 
491
534
  Parameters
492
535
  ----------
493
- packages : Dict[str, str], default {}
536
+ packages : Dict[str, str], default: {}
494
537
  Packages to use for this step. The key is the name of the package
495
538
  and the value is the version to use.
496
- libraries : Dict[str, str], default {}
497
- Supported for backward compatibility. When used with packages, packages will take precedence.
498
- python : str, optional, default None
539
+ python : str, optional, default: None
499
540
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
500
541
  that the version used will correspond to the version of the Python interpreter used to start the run.
501
- disabled : bool, default False
502
- If set to True, disables @conda.
503
542
  """
504
543
  ...
505
544
 
@@ -555,325 +594,200 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
555
594
  ...
556
595
 
557
596
  @typing.overload
558
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
597
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
559
598
  """
560
- Specifies the number of times the task corresponding
561
- to a step needs to be retried.
562
-
563
- This decorator is useful for handling transient errors, such as networking issues.
564
- If your task contains operations that can't be retried safely, e.g. database updates,
565
- it is advisable to annotate it with `@retry(times=0)`.
599
+ Decorator prototype for all step decorators. This function gets specialized
600
+ and imported for all decorators types by _import_plugin_decorators().
601
+ """
602
+ ...
603
+
604
+ @typing.overload
605
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
606
+ ...
607
+
608
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
609
+ """
610
+ Decorator prototype for all step decorators. This function gets specialized
611
+ and imported for all decorators types by _import_plugin_decorators().
612
+ """
613
+ ...
614
+
615
+ @typing.overload
616
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
617
+ """
618
+ Creates a human-readable report, a Metaflow Card, after this step completes.
566
619
 
567
- This can be used in conjunction with the `@catch` decorator. The `@catch`
568
- decorator will execute a no-op task after all retries have been exhausted,
569
- ensuring that the flow execution can continue.
620
+ Note that you may add multiple `@card` decorators in a step with different parameters.
570
621
 
571
622
 
572
623
  Parameters
573
624
  ----------
574
- times : int, default 3
575
- Number of times to retry this task.
576
- minutes_between_retries : int, default 2
577
- Number of minutes between retries.
625
+ type : str, default 'default'
626
+ Card type.
627
+ id : str, optional, default None
628
+ If multiple cards are present, use this id to identify this card.
629
+ options : Dict[str, Any], default {}
630
+ Options passed to the card. The contents depend on the card type.
631
+ timeout : int, default 45
632
+ Interrupt reporting if it takes more than this many seconds.
578
633
  """
579
634
  ...
580
635
 
581
636
  @typing.overload
582
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
637
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
583
638
  ...
584
639
 
585
640
  @typing.overload
586
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
641
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
587
642
  ...
588
643
 
589
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
644
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
590
645
  """
591
- Specifies the number of times the task corresponding
592
- to a step needs to be retried.
593
-
594
- This decorator is useful for handling transient errors, such as networking issues.
595
- If your task contains operations that can't be retried safely, e.g. database updates,
596
- it is advisable to annotate it with `@retry(times=0)`.
646
+ Creates a human-readable report, a Metaflow Card, after this step completes.
597
647
 
598
- This can be used in conjunction with the `@catch` decorator. The `@catch`
599
- decorator will execute a no-op task after all retries have been exhausted,
600
- ensuring that the flow execution can continue.
648
+ Note that you may add multiple `@card` decorators in a step with different parameters.
601
649
 
602
650
 
603
651
  Parameters
604
652
  ----------
605
- times : int, default 3
606
- Number of times to retry this task.
607
- minutes_between_retries : int, default 2
608
- Number of minutes between retries.
653
+ type : str, default 'default'
654
+ Card type.
655
+ id : str, optional, default None
656
+ If multiple cards are present, use this id to identify this card.
657
+ options : Dict[str, Any], default {}
658
+ Options passed to the card. The contents depend on the card type.
659
+ timeout : int, default 45
660
+ Interrupt reporting if it takes more than this many seconds.
609
661
  """
610
662
  ...
611
663
 
612
664
  @typing.overload
613
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
665
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
614
666
  """
615
- Specifies the PyPI packages for the step.
667
+ Enables loading / saving of models within a step.
616
668
 
617
- Information in this decorator will augment any
618
- attributes set in the `@pyi_base` flow-level decorator. Hence,
619
- you can use `@pypi_base` to set packages required by all
620
- steps and use `@pypi` to specify step-specific overrides.
621
669
 
622
670
 
623
671
  Parameters
624
672
  ----------
625
- packages : Dict[str, str], default: {}
626
- Packages to use for this step. The key is the name of the package
627
- and the value is the version to use.
628
- python : str, optional, default: None
629
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
630
- that the version used will correspond to the version of the Python interpreter used to start the run.
673
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
674
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
675
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
676
+ - `current.checkpoint`
677
+ - `current.model`
678
+ - `current.huggingface_hub`
679
+
680
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
681
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
682
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
683
+
684
+ temp_dir_root : str, default: None
685
+ The root directory under which `current.model.loaded` will store loaded models
631
686
  """
632
687
  ...
633
688
 
634
689
  @typing.overload
635
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
690
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
636
691
  ...
637
692
 
638
693
  @typing.overload
639
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
694
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
640
695
  ...
641
696
 
642
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
697
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
643
698
  """
644
- Specifies the PyPI packages for the step.
699
+ Enables loading / saving of models within a step.
645
700
 
646
- Information in this decorator will augment any
647
- attributes set in the `@pyi_base` flow-level decorator. Hence,
648
- you can use `@pypi_base` to set packages required by all
649
- steps and use `@pypi` to specify step-specific overrides.
650
701
 
651
702
 
652
703
  Parameters
653
704
  ----------
654
- packages : Dict[str, str], default: {}
655
- Packages to use for this step. The key is the name of the package
656
- and the value is the version to use.
657
- python : str, optional, default: None
658
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
659
- that the version used will correspond to the version of the Python interpreter used to start the run.
660
- """
661
- ...
662
-
663
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
664
- """
665
- Specifies that this step should execute on DGX cloud.
666
-
667
-
668
- Parameters
669
- ----------
670
- gpu : int
671
- Number of GPUs to use.
672
- gpu_type : str
673
- Type of Nvidia GPU to use.
674
- """
675
- ...
676
-
677
- def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
678
- """
679
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
680
-
681
- User code call
682
- -----------
683
- @ollama(
684
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
685
- backend='local'
686
- )
687
-
688
- Valid backend options
689
- ---------------------
690
- - 'local': Run as a separate process on the local task machine.
691
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
692
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
693
-
694
- Valid model options
695
- ----------------
696
- - 'llama3.2'
697
- - 'llama3.3'
698
- - any model here https://ollama.com/search
699
-
700
-
701
- Parameters
702
- ----------
703
- models: list[Ollama]
704
- List of Ollama containers running models in sidecars.
705
- backend: str
706
- Determines where and how to run the Ollama process.
707
- """
708
- ...
709
-
710
- @typing.overload
711
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
712
- """
713
- Specifies the resources needed when executing this step.
714
-
715
- Use `@resources` to specify the resource requirements
716
- independently of the specific compute layer (`@batch`, `@kubernetes`).
717
-
718
- You can choose the compute layer on the command line by executing e.g.
719
- ```
720
- python myflow.py run --with batch
721
- ```
722
- or
723
- ```
724
- python myflow.py run --with kubernetes
725
- ```
726
- which executes the flow on the desired system using the
727
- requirements specified in `@resources`.
728
-
729
-
730
- Parameters
731
- ----------
732
- cpu : int, default 1
733
- Number of CPUs required for this step.
734
- gpu : int, optional, default None
735
- Number of GPUs required for this step.
736
- disk : int, optional, default None
737
- Disk size (in MB) required for this step. Only applies on Kubernetes.
738
- memory : int, default 4096
739
- Memory size (in MB) required for this step.
740
- shared_memory : int, optional, default None
741
- The value for the size (in MiB) of the /dev/shm volume for this step.
742
- This parameter maps to the `--shm-size` option in Docker.
743
- """
744
- ...
745
-
746
- @typing.overload
747
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
748
- ...
749
-
750
- @typing.overload
751
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
752
- ...
753
-
754
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
755
- """
756
- Specifies the resources needed when executing this step.
757
-
758
- Use `@resources` to specify the resource requirements
759
- independently of the specific compute layer (`@batch`, `@kubernetes`).
760
-
761
- You can choose the compute layer on the command line by executing e.g.
762
- ```
763
- python myflow.py run --with batch
764
- ```
765
- or
766
- ```
767
- python myflow.py run --with kubernetes
768
- ```
769
- which executes the flow on the desired system using the
770
- requirements specified in `@resources`.
705
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
706
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
707
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
708
+ - `current.checkpoint`
709
+ - `current.model`
710
+ - `current.huggingface_hub`
771
711
 
712
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
713
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
714
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
772
715
 
773
- Parameters
774
- ----------
775
- cpu : int, default 1
776
- Number of CPUs required for this step.
777
- gpu : int, optional, default None
778
- Number of GPUs required for this step.
779
- disk : int, optional, default None
780
- Disk size (in MB) required for this step. Only applies on Kubernetes.
781
- memory : int, default 4096
782
- Memory size (in MB) required for this step.
783
- shared_memory : int, optional, default None
784
- The value for the size (in MiB) of the /dev/shm volume for this step.
785
- This parameter maps to the `--shm-size` option in Docker.
786
- """
787
- ...
788
-
789
- @typing.overload
790
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
791
- """
792
- Decorator prototype for all step decorators. This function gets specialized
793
- and imported for all decorators types by _import_plugin_decorators().
794
- """
795
- ...
796
-
797
- @typing.overload
798
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
799
- ...
800
-
801
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
802
- """
803
- Decorator prototype for all step decorators. This function gets specialized
804
- and imported for all decorators types by _import_plugin_decorators().
716
+ temp_dir_root : str, default: None
717
+ The root directory under which `current.model.loaded` will store loaded models
805
718
  """
806
719
  ...
807
720
 
808
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
721
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
809
722
  """
810
- Specifies that this step is used to deploy an instance of the app.
811
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
723
+ Specifies that this step should execute on DGX cloud.
812
724
 
813
725
 
814
726
  Parameters
815
727
  ----------
816
- app_port : int
728
+ gpu : int
817
729
  Number of GPUs to use.
818
- app_name : str
819
- Name of the app to deploy.
730
+ gpu_type : str
731
+ Type of Nvidia GPU to use.
820
732
  """
821
733
  ...
822
734
 
823
735
  @typing.overload
824
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
736
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
825
737
  """
826
- Enables checkpointing for a step.
738
+ Specifies the Conda environment for the step.
827
739
 
740
+ Information in this decorator will augment any
741
+ attributes set in the `@conda_base` flow-level decorator. Hence,
742
+ you can use `@conda_base` to set packages required by all
743
+ steps and use `@conda` to specify step-specific overrides.
828
744
 
829
745
 
830
746
  Parameters
831
747
  ----------
832
- load_policy : str, default: "fresh"
833
- The policy for loading the checkpoint. The following policies are supported:
834
- - "eager": Loads the the latest available checkpoint within the namespace.
835
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
836
- will be loaded at the start of the task.
837
- - "none": Do not load any checkpoint
838
- - "fresh": Loads the lastest checkpoint created within the running Task.
839
- This mode helps loading checkpoints across various retry attempts of the same task.
840
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
841
- created within the task will be loaded when the task is retries execution on failure.
842
-
843
- temp_dir_root : str, default: None
844
- The root directory under which `current.checkpoint.directory` will be created.
748
+ packages : Dict[str, str], default {}
749
+ Packages to use for this step. The key is the name of the package
750
+ and the value is the version to use.
751
+ libraries : Dict[str, str], default {}
752
+ Supported for backward compatibility. When used with packages, packages will take precedence.
753
+ python : str, optional, default None
754
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
755
+ that the version used will correspond to the version of the Python interpreter used to start the run.
756
+ disabled : bool, default False
757
+ If set to True, disables @conda.
845
758
  """
846
759
  ...
847
760
 
848
761
  @typing.overload
849
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
762
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
850
763
  ...
851
764
 
852
765
  @typing.overload
853
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
766
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
854
767
  ...
855
768
 
856
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
769
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
857
770
  """
858
- Enables checkpointing for a step.
771
+ Specifies the Conda environment for the step.
859
772
 
773
+ Information in this decorator will augment any
774
+ attributes set in the `@conda_base` flow-level decorator. Hence,
775
+ you can use `@conda_base` to set packages required by all
776
+ steps and use `@conda` to specify step-specific overrides.
860
777
 
861
778
 
862
779
  Parameters
863
780
  ----------
864
- load_policy : str, default: "fresh"
865
- The policy for loading the checkpoint. The following policies are supported:
866
- - "eager": Loads the the latest available checkpoint within the namespace.
867
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
868
- will be loaded at the start of the task.
869
- - "none": Do not load any checkpoint
870
- - "fresh": Loads the lastest checkpoint created within the running Task.
871
- This mode helps loading checkpoints across various retry attempts of the same task.
872
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
873
- created within the task will be loaded when the task is retries execution on failure.
874
-
875
- temp_dir_root : str, default: None
876
- The root directory under which `current.checkpoint.directory` will be created.
781
+ packages : Dict[str, str], default {}
782
+ Packages to use for this step. The key is the name of the package
783
+ and the value is the version to use.
784
+ libraries : Dict[str, str], default {}
785
+ Supported for backward compatibility. When used with packages, packages will take precedence.
786
+ python : str, optional, default None
787
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
788
+ that the version used will correspond to the version of the Python interpreter used to start the run.
789
+ disabled : bool, default False
790
+ If set to True, disables @conda.
877
791
  """
878
792
  ...
879
793
 
@@ -962,6 +876,217 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
962
876
  """
963
877
  ...
964
878
 
879
+ @typing.overload
880
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
881
+ """
882
+ Internal decorator to support Fast bakery
883
+ """
884
+ ...
885
+
886
+ @typing.overload
887
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
888
+ ...
889
+
890
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
891
+ """
892
+ Internal decorator to support Fast bakery
893
+ """
894
+ ...
895
+
896
+ @typing.overload
897
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
898
+ """
899
+ Enables checkpointing for a step.
900
+
901
+
902
+
903
+ Parameters
904
+ ----------
905
+ load_policy : str, default: "fresh"
906
+ The policy for loading the checkpoint. The following policies are supported:
907
+ - "eager": Loads the the latest available checkpoint within the namespace.
908
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
909
+ will be loaded at the start of the task.
910
+ - "none": Do not load any checkpoint
911
+ - "fresh": Loads the lastest checkpoint created within the running Task.
912
+ This mode helps loading checkpoints across various retry attempts of the same task.
913
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
914
+ created within the task will be loaded when the task is retries execution on failure.
915
+
916
+ temp_dir_root : str, default: None
917
+ The root directory under which `current.checkpoint.directory` will be created.
918
+ """
919
+ ...
920
+
921
+ @typing.overload
922
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
923
+ ...
924
+
925
+ @typing.overload
926
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
927
+ ...
928
+
929
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
930
+ """
931
+ Enables checkpointing for a step.
932
+
933
+
934
+
935
+ Parameters
936
+ ----------
937
+ load_policy : str, default: "fresh"
938
+ The policy for loading the checkpoint. The following policies are supported:
939
+ - "eager": Loads the the latest available checkpoint within the namespace.
940
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
941
+ will be loaded at the start of the task.
942
+ - "none": Do not load any checkpoint
943
+ - "fresh": Loads the lastest checkpoint created within the running Task.
944
+ This mode helps loading checkpoints across various retry attempts of the same task.
945
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
946
+ created within the task will be loaded when the task is retries execution on failure.
947
+
948
+ temp_dir_root : str, default: None
949
+ The root directory under which `current.checkpoint.directory` will be created.
950
+ """
951
+ ...
952
+
953
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
954
+ """
955
+ Specifies that this step should execute on DGX cloud.
956
+
957
+
958
+ Parameters
959
+ ----------
960
+ gpu : int
961
+ Number of GPUs to use.
962
+ gpu_type : str
963
+ Type of Nvidia GPU to use.
964
+ queue_timeout : int
965
+ Time to keep the job in NVCF's queue.
966
+ """
967
+ ...
968
+
969
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
970
+ """
971
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
972
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
973
+
974
+
975
+ Parameters
976
+ ----------
977
+ timeout : int
978
+ Time, in seconds before the task times out and fails. (Default: 3600)
979
+ poke_interval : int
980
+ Time in seconds that the job should wait in between each try. (Default: 60)
981
+ mode : str
982
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
983
+ exponential_backoff : bool
984
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
985
+ pool : str
986
+ the slot pool this task should run in,
987
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
988
+ soft_fail : bool
989
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
990
+ name : str
991
+ Name of the sensor on Airflow
992
+ description : str
993
+ Description of sensor in the Airflow UI
994
+ external_dag_id : str
995
+ The dag_id that contains the task you want to wait for.
996
+ external_task_ids : List[str]
997
+ The list of task_ids that you want to wait for.
998
+ If None (default value) the sensor waits for the DAG. (Default: None)
999
+ allowed_states : List[str]
1000
+ Iterable of allowed states, (Default: ['success'])
1001
+ failed_states : List[str]
1002
+ Iterable of failed or dis-allowed states. (Default: None)
1003
+ execution_delta : datetime.timedelta
1004
+ time difference with the previous execution to look at,
1005
+ the default is the same logical date as the current task or DAG. (Default: None)
1006
+ check_existence: bool
1007
+ Set to True to check if the external task exists or check if
1008
+ the DAG to wait for exists. (Default: True)
1009
+ """
1010
+ ...
1011
+
1012
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1013
+ """
1014
+ Specifies what flows belong to the same project.
1015
+
1016
+ A project-specific namespace is created for all flows that
1017
+ use the same `@project(name)`.
1018
+
1019
+
1020
+ Parameters
1021
+ ----------
1022
+ name : str
1023
+ Project name. Make sure that the name is unique amongst all
1024
+ projects that use the same production scheduler. The name may
1025
+ contain only lowercase alphanumeric characters and underscores.
1026
+
1027
+ branch : Optional[str], default None
1028
+ The branch to use. If not specified, the branch is set to
1029
+ `user.<username>` unless `production` is set to `True`. This can
1030
+ also be set on the command line using `--branch` as a top-level option.
1031
+ It is an error to specify `branch` in the decorator and on the command line.
1032
+
1033
+ production : bool, default False
1034
+ Whether or not the branch is the production branch. This can also be set on the
1035
+ command line using `--production` as a top-level option. It is an error to specify
1036
+ `production` in the decorator and on the command line.
1037
+ The project branch name will be:
1038
+ - if `branch` is specified:
1039
+ - if `production` is True: `prod.<branch>`
1040
+ - if `production` is False: `test.<branch>`
1041
+ - if `branch` is not specified:
1042
+ - if `production` is True: `prod`
1043
+ - if `production` is False: `user.<username>`
1044
+ """
1045
+ ...
1046
+
1047
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1048
+ """
1049
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1050
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1051
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1052
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1053
+ starts only after all sensors finish.
1054
+
1055
+
1056
+ Parameters
1057
+ ----------
1058
+ timeout : int
1059
+ Time, in seconds before the task times out and fails. (Default: 3600)
1060
+ poke_interval : int
1061
+ Time in seconds that the job should wait in between each try. (Default: 60)
1062
+ mode : str
1063
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1064
+ exponential_backoff : bool
1065
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1066
+ pool : str
1067
+ the slot pool this task should run in,
1068
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1069
+ soft_fail : bool
1070
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1071
+ name : str
1072
+ Name of the sensor on Airflow
1073
+ description : str
1074
+ Description of sensor in the Airflow UI
1075
+ bucket_key : Union[str, List[str]]
1076
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1077
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1078
+ bucket_name : str
1079
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1080
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1081
+ wildcard_match : bool
1082
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1083
+ aws_conn_id : str
1084
+ a reference to the s3 connection on Airflow. (Default: None)
1085
+ verify : bool
1086
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1087
+ """
1088
+ ...
1089
+
965
1090
  @typing.overload
966
1091
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
967
1092
  """
@@ -1055,6 +1180,57 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1055
1180
  """
1056
1181
  ...
1057
1182
 
1183
+ @typing.overload
1184
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1185
+ """
1186
+ Specifies the Conda environment for all steps of the flow.
1187
+
1188
+ Use `@conda_base` to set common libraries required by all
1189
+ steps and use `@conda` to specify step-specific additions.
1190
+
1191
+
1192
+ Parameters
1193
+ ----------
1194
+ packages : Dict[str, str], default {}
1195
+ Packages to use for this flow. The key is the name of the package
1196
+ and the value is the version to use.
1197
+ libraries : Dict[str, str], default {}
1198
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1199
+ python : str, optional, default None
1200
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1201
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1202
+ disabled : bool, default False
1203
+ If set to True, disables Conda.
1204
+ """
1205
+ ...
1206
+
1207
+ @typing.overload
1208
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1209
+ ...
1210
+
1211
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1212
+ """
1213
+ Specifies the Conda environment for all steps of the flow.
1214
+
1215
+ Use `@conda_base` to set common libraries required by all
1216
+ steps and use `@conda` to specify step-specific additions.
1217
+
1218
+
1219
+ Parameters
1220
+ ----------
1221
+ packages : Dict[str, str], default {}
1222
+ Packages to use for this flow. The key is the name of the package
1223
+ and the value is the version to use.
1224
+ libraries : Dict[str, str], default {}
1225
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1226
+ python : str, optional, default None
1227
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1228
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1229
+ disabled : bool, default False
1230
+ If set to True, disables Conda.
1231
+ """
1232
+ ...
1233
+
1058
1234
  def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1059
1235
  """
1060
1236
  Allows setting external datastores to save data for the
@@ -1169,49 +1345,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1169
1345
  """
1170
1346
  ...
1171
1347
 
1172
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1173
- """
1174
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1175
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1176
-
1177
-
1178
- Parameters
1179
- ----------
1180
- timeout : int
1181
- Time, in seconds before the task times out and fails. (Default: 3600)
1182
- poke_interval : int
1183
- Time in seconds that the job should wait in between each try. (Default: 60)
1184
- mode : str
1185
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1186
- exponential_backoff : bool
1187
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1188
- pool : str
1189
- the slot pool this task should run in,
1190
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1191
- soft_fail : bool
1192
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1193
- name : str
1194
- Name of the sensor on Airflow
1195
- description : str
1196
- Description of sensor in the Airflow UI
1197
- external_dag_id : str
1198
- The dag_id that contains the task you want to wait for.
1199
- external_task_ids : List[str]
1200
- The list of task_ids that you want to wait for.
1201
- If None (default value) the sensor waits for the DAG. (Default: None)
1202
- allowed_states : List[str]
1203
- Iterable of allowed states, (Default: ['success'])
1204
- failed_states : List[str]
1205
- Iterable of failed or dis-allowed states. (Default: None)
1206
- execution_delta : datetime.timedelta
1207
- time difference with the previous execution to look at,
1208
- the default is the same logical date as the current task or DAG. (Default: None)
1209
- check_existence: bool
1210
- Set to True to check if the external task exists or check if
1211
- the DAG to wait for exists. (Default: True)
1212
- """
1213
- ...
1214
-
1215
1348
  @typing.overload
1216
1349
  def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1217
1350
  """
@@ -1253,49 +1386,6 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
1253
1386
  """
1254
1387
  ...
1255
1388
 
1256
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1257
- """
1258
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1259
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1260
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1261
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1262
- starts only after all sensors finish.
1263
-
1264
-
1265
- Parameters
1266
- ----------
1267
- timeout : int
1268
- Time, in seconds before the task times out and fails. (Default: 3600)
1269
- poke_interval : int
1270
- Time in seconds that the job should wait in between each try. (Default: 60)
1271
- mode : str
1272
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1273
- exponential_backoff : bool
1274
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1275
- pool : str
1276
- the slot pool this task should run in,
1277
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1278
- soft_fail : bool
1279
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1280
- name : str
1281
- Name of the sensor on Airflow
1282
- description : str
1283
- Description of sensor in the Airflow UI
1284
- bucket_key : Union[str, List[str]]
1285
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1286
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1287
- bucket_name : str
1288
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1289
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1290
- wildcard_match : bool
1291
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1292
- aws_conn_id : str
1293
- a reference to the s3 connection on Airflow. (Default: None)
1294
- verify : bool
1295
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1296
- """
1297
- ...
1298
-
1299
1389
  @typing.overload
1300
1390
  def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1301
1391
  """
@@ -1448,91 +1538,5 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1448
1538
  """
1449
1539
  ...
1450
1540
 
1451
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1452
- """
1453
- Specifies what flows belong to the same project.
1454
-
1455
- A project-specific namespace is created for all flows that
1456
- use the same `@project(name)`.
1457
-
1458
-
1459
- Parameters
1460
- ----------
1461
- name : str
1462
- Project name. Make sure that the name is unique amongst all
1463
- projects that use the same production scheduler. The name may
1464
- contain only lowercase alphanumeric characters and underscores.
1465
-
1466
- branch : Optional[str], default None
1467
- The branch to use. If not specified, the branch is set to
1468
- `user.<username>` unless `production` is set to `True`. This can
1469
- also be set on the command line using `--branch` as a top-level option.
1470
- It is an error to specify `branch` in the decorator and on the command line.
1471
-
1472
- production : bool, default False
1473
- Whether or not the branch is the production branch. This can also be set on the
1474
- command line using `--production` as a top-level option. It is an error to specify
1475
- `production` in the decorator and on the command line.
1476
- The project branch name will be:
1477
- - if `branch` is specified:
1478
- - if `production` is True: `prod.<branch>`
1479
- - if `production` is False: `test.<branch>`
1480
- - if `branch` is not specified:
1481
- - if `production` is True: `prod`
1482
- - if `production` is False: `user.<username>`
1483
- """
1484
- ...
1485
-
1486
- @typing.overload
1487
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1488
- """
1489
- Specifies the Conda environment for all steps of the flow.
1490
-
1491
- Use `@conda_base` to set common libraries required by all
1492
- steps and use `@conda` to specify step-specific additions.
1493
-
1494
-
1495
- Parameters
1496
- ----------
1497
- packages : Dict[str, str], default {}
1498
- Packages to use for this flow. The key is the name of the package
1499
- and the value is the version to use.
1500
- libraries : Dict[str, str], default {}
1501
- Supported for backward compatibility. When used with packages, packages will take precedence.
1502
- python : str, optional, default None
1503
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1504
- that the version used will correspond to the version of the Python interpreter used to start the run.
1505
- disabled : bool, default False
1506
- If set to True, disables Conda.
1507
- """
1508
- ...
1509
-
1510
- @typing.overload
1511
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1512
- ...
1513
-
1514
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1515
- """
1516
- Specifies the Conda environment for all steps of the flow.
1517
-
1518
- Use `@conda_base` to set common libraries required by all
1519
- steps and use `@conda` to specify step-specific additions.
1520
-
1521
-
1522
- Parameters
1523
- ----------
1524
- packages : Dict[str, str], default {}
1525
- Packages to use for this flow. The key is the name of the package
1526
- and the value is the version to use.
1527
- libraries : Dict[str, str], default {}
1528
- Supported for backward compatibility. When used with packages, packages will take precedence.
1529
- python : str, optional, default None
1530
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1531
- that the version used will correspond to the version of the Python interpreter used to start the run.
1532
- disabled : bool, default False
1533
- If set to True, disables Conda.
1534
- """
1535
- ...
1536
-
1537
1541
  pkg_name: str
1538
1542