ob-metaflow-stubs 6.0.10.19__py2.py3-none-any.whl → 6.0.10.20__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow-stubs might be problematic. Click here for more details.

Files changed (266) hide show
  1. metaflow-stubs/__init__.pyi +1025 -1024
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +3 -2
  7. metaflow-stubs/client/core.pyi +39 -7
  8. metaflow-stubs/client/filecache.pyi +21 -5
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +3 -3
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +16 -2
  20. metaflow-stubs/metaflow_current.pyi +58 -58
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/__init__.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/hf_hub_card.pyi +3 -3
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  64. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  65. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +3 -3
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +3 -3
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +4 -4
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +3 -3
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  110. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  111. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  112. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  113. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  114. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +3 -3
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  116. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  117. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
  118. metaflow-stubs/multicore_utils.pyi +2 -2
  119. metaflow-stubs/ob_internal.pyi +2 -2
  120. metaflow-stubs/packaging_sys/__init__.pyi +5 -5
  121. metaflow-stubs/packaging_sys/backend.pyi +4 -4
  122. metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
  123. metaflow-stubs/packaging_sys/tar_backend.pyi +6 -6
  124. metaflow-stubs/packaging_sys/utils.pyi +2 -5
  125. metaflow-stubs/packaging_sys/v1.pyi +3 -3
  126. metaflow-stubs/parameters.pyi +3 -3
  127. metaflow-stubs/plugins/__init__.pyi +12 -12
  128. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  129. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  130. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  131. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  132. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  133. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  134. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  135. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  136. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  137. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  138. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  139. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  140. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  141. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  142. metaflow-stubs/plugins/argo/exit_hooks.pyi +2 -2
  143. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  144. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  145. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  146. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  147. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  148. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  149. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  150. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +5 -5
  152. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  153. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  154. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  155. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  157. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  158. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  159. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  160. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  161. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  162. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +5 -5
  163. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  164. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  165. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  166. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  167. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  168. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  169. metaflow-stubs/plugins/cards/card_datastore.pyi +4 -2
  170. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  171. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  172. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  173. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  174. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  175. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  176. metaflow-stubs/plugins/cards/card_modules/json_viewer.pyi +2 -2
  177. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  178. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  179. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  180. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  181. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  182. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  183. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  184. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  185. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  186. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -3
  187. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  188. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  189. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  190. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  191. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  192. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  193. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  194. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  195. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  196. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  197. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  198. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +5 -5
  199. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  200. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  201. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  202. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  203. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  204. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  205. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  206. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  207. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  208. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  209. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  210. metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
  211. metaflow-stubs/plugins/optuna/__init__.pyi +2 -2
  212. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  213. metaflow-stubs/plugins/parsers.pyi +2 -2
  214. metaflow-stubs/plugins/perimeters.pyi +2 -2
  215. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  216. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  217. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  218. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  219. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  220. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  221. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  222. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  223. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  224. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  225. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  226. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  227. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  228. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  229. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  230. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  231. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  232. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  233. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  234. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  235. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  236. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  237. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  238. metaflow-stubs/profilers/__init__.pyi +2 -2
  239. metaflow-stubs/pylint_wrapper.pyi +2 -2
  240. metaflow-stubs/runner/__init__.pyi +2 -2
  241. metaflow-stubs/runner/deployer.pyi +32 -32
  242. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  243. metaflow-stubs/runner/metaflow_runner.pyi +129 -16
  244. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  245. metaflow-stubs/runner/nbrun.pyi +2 -2
  246. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  247. metaflow-stubs/runner/utils.pyi +3 -3
  248. metaflow-stubs/system/__init__.pyi +2 -2
  249. metaflow-stubs/system/system_logger.pyi +3 -3
  250. metaflow-stubs/system/system_monitor.pyi +2 -2
  251. metaflow-stubs/tagging_util.pyi +2 -2
  252. metaflow-stubs/tuple_util.pyi +2 -2
  253. metaflow-stubs/user_configs/__init__.pyi +2 -2
  254. metaflow-stubs/user_configs/config_options.pyi +3 -3
  255. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  256. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  257. metaflow-stubs/user_decorators/common.pyi +2 -2
  258. metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
  259. metaflow-stubs/user_decorators/mutable_step.pyi +4 -4
  260. metaflow-stubs/user_decorators/user_flow_decorator.pyi +2 -2
  261. metaflow-stubs/user_decorators/user_step_decorator.pyi +14 -5
  262. {ob_metaflow_stubs-6.0.10.19.dist-info → ob_metaflow_stubs-6.0.10.20.dist-info}/METADATA +1 -1
  263. ob_metaflow_stubs-6.0.10.20.dist-info/RECORD +266 -0
  264. ob_metaflow_stubs-6.0.10.19.dist-info/RECORD +0 -266
  265. {ob_metaflow_stubs-6.0.10.19.dist-info → ob_metaflow_stubs-6.0.10.20.dist-info}/WHEEL +0 -0
  266. {ob_metaflow_stubs-6.0.10.19.dist-info → ob_metaflow_stubs-6.0.10.20.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.18.13.1+obcheckpoint(0.2.8);ob(v1) #
4
- # Generated on 2025-10-21T09:01:27.410960 #
3
+ # MF version: 2.19.3.1+obcheckpoint(0.2.8);ob(v1) #
4
+ # Generated on 2025-10-28T11:23:52.528703 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -39,19 +39,19 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
39
39
  from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
- from . import events as events
43
42
  from . import metaflow_git as metaflow_git
44
43
  from . import cards as cards
44
+ from . import events as events
45
45
  from . import tuple_util as tuple_util
46
46
  from . import runner as runner
47
47
  from . import plugins as plugins
48
48
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
49
49
  from . import includefile as includefile
50
50
  from .includefile import IncludeFile as IncludeFile
51
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
52
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
53
51
  from .plugins.parsers import yaml_parser as yaml_parser
54
52
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
53
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
54
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
55
55
  from . import client as client
56
56
  from .client.core import namespace as namespace
57
57
  from .client.core import get_namespace as get_namespace
@@ -59,6 +59,7 @@ from .client.core import default_namespace as default_namespace
59
59
  from .client.core import metadata as metadata
60
60
  from .client.core import get_metadata as get_metadata
61
61
  from .client.core import default_metadata as default_metadata
62
+ from .client.core import inspect_spin as inspect_spin
62
63
  from .client.core import Metaflow as Metaflow
63
64
  from .client.core import Flow as Flow
64
65
  from .client.core import Run as Run
@@ -85,8 +86,8 @@ from .mf_extensions.outerbounds.plugins.checkpoint_datastores.nebius import nebi
85
86
  from .mf_extensions.outerbounds.plugins.checkpoint_datastores.coreweave import coreweave_checkpoints as coreweave_checkpoints
86
87
  from .mf_extensions.outerbounds.plugins.aws.assume_role_decorator import assume_role as assume_role
87
88
  from .mf_extensions.outerbounds.plugins.apps.core.deployer import AppDeployer as AppDeployer
88
- from . import system as system
89
89
  from . import cli_components as cli_components
90
+ from . import system as system
90
91
  from . import pylint_wrapper as pylint_wrapper
91
92
  from . import cli as cli
92
93
  from . import profilers as profilers
@@ -169,137 +170,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
169
170
  """
170
171
  ...
171
172
 
172
- @typing.overload
173
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
174
- """
175
- Specifies the resources needed when executing this step.
176
-
177
- Use `@resources` to specify the resource requirements
178
- independently of the specific compute layer (`@batch`, `@kubernetes`).
179
-
180
- You can choose the compute layer on the command line by executing e.g.
181
- ```
182
- python myflow.py run --with batch
183
- ```
184
- or
185
- ```
186
- python myflow.py run --with kubernetes
187
- ```
188
- which executes the flow on the desired system using the
189
- requirements specified in `@resources`.
190
-
191
-
192
- Parameters
193
- ----------
194
- cpu : int, default 1
195
- Number of CPUs required for this step.
196
- gpu : int, optional, default None
197
- Number of GPUs required for this step.
198
- disk : int, optional, default None
199
- Disk size (in MB) required for this step. Only applies on Kubernetes.
200
- memory : int, default 4096
201
- Memory size (in MB) required for this step.
202
- shared_memory : int, optional, default None
203
- The value for the size (in MiB) of the /dev/shm volume for this step.
204
- This parameter maps to the `--shm-size` option in Docker.
205
- """
206
- ...
207
-
208
- @typing.overload
209
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
210
- ...
211
-
212
- @typing.overload
213
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
214
- ...
215
-
216
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
217
- """
218
- Specifies the resources needed when executing this step.
219
-
220
- Use `@resources` to specify the resource requirements
221
- independently of the specific compute layer (`@batch`, `@kubernetes`).
222
-
223
- You can choose the compute layer on the command line by executing e.g.
224
- ```
225
- python myflow.py run --with batch
226
- ```
227
- or
228
- ```
229
- python myflow.py run --with kubernetes
230
- ```
231
- which executes the flow on the desired system using the
232
- requirements specified in `@resources`.
233
-
234
-
235
- Parameters
236
- ----------
237
- cpu : int, default 1
238
- Number of CPUs required for this step.
239
- gpu : int, optional, default None
240
- Number of GPUs required for this step.
241
- disk : int, optional, default None
242
- Disk size (in MB) required for this step. Only applies on Kubernetes.
243
- memory : int, default 4096
244
- Memory size (in MB) required for this step.
245
- shared_memory : int, optional, default None
246
- The value for the size (in MiB) of the /dev/shm volume for this step.
247
- This parameter maps to the `--shm-size` option in Docker.
248
- """
249
- ...
250
-
251
- @typing.overload
252
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
253
- """
254
- Specifies environment variables to be set prior to the execution of a step.
255
-
256
-
257
- Parameters
258
- ----------
259
- vars : Dict[str, str], default {}
260
- Dictionary of environment variables to set.
261
- """
262
- ...
263
-
264
- @typing.overload
265
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
266
- ...
267
-
268
- @typing.overload
269
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
270
- ...
271
-
272
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
273
- """
274
- Specifies environment variables to be set prior to the execution of a step.
275
-
276
-
277
- Parameters
278
- ----------
279
- vars : Dict[str, str], default {}
280
- Dictionary of environment variables to set.
281
- """
282
- ...
283
-
284
- @typing.overload
285
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
286
- """
287
- A simple decorator that demonstrates using CardDecoratorInjector
288
- to inject a card and render simple markdown content.
289
- """
290
- ...
291
-
292
- @typing.overload
293
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
294
- ...
295
-
296
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
297
- """
298
- A simple decorator that demonstrates using CardDecoratorInjector
299
- to inject a card and render simple markdown content.
300
- """
301
- ...
302
-
303
173
  @typing.overload
304
174
  def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
305
175
  """
@@ -359,63 +229,109 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
359
229
  """
360
230
  ...
361
231
 
362
- @typing.overload
363
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
232
+ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
364
233
  """
365
- Specifies the number of times the task corresponding
366
- to a step needs to be retried.
234
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
367
235
 
368
- This decorator is useful for handling transient errors, such as networking issues.
369
- If your task contains operations that can't be retried safely, e.g. database updates,
370
- it is advisable to annotate it with `@retry(times=0)`.
236
+ User code call
237
+ --------------
238
+ @vllm(
239
+ model="...",
240
+ ...
241
+ )
371
242
 
372
- This can be used in conjunction with the `@catch` decorator. The `@catch`
373
- decorator will execute a no-op task after all retries have been exhausted,
374
- ensuring that the flow execution can continue.
243
+ Valid backend options
244
+ ---------------------
245
+ - 'local': Run as a separate process on the local task machine.
246
+
247
+ Valid model options
248
+ -------------------
249
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
250
+
251
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
252
+ If you need multiple models, you must create multiple @vllm decorators.
375
253
 
376
254
 
377
255
  Parameters
378
256
  ----------
379
- times : int, default 3
380
- Number of times to retry this task.
381
- minutes_between_retries : int, default 2
382
- Number of minutes between retries.
257
+ model: str
258
+ HuggingFace model identifier to be served by vLLM.
259
+ backend: str
260
+ Determines where and how to run the vLLM process.
261
+ openai_api_server: bool
262
+ Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
263
+ Default is False (uses native engine).
264
+ Set to True for backward compatibility with existing code.
265
+ debug: bool
266
+ Whether to turn on verbose debugging logs.
267
+ card_refresh_interval: int
268
+ Interval in seconds for refreshing the vLLM status card.
269
+ Only used when openai_api_server=True.
270
+ max_retries: int
271
+ Maximum number of retries checking for vLLM server startup.
272
+ Only used when openai_api_server=True.
273
+ retry_alert_frequency: int
274
+ Frequency of alert logs for vLLM server startup retries.
275
+ Only used when openai_api_server=True.
276
+ engine_args : dict
277
+ Additional keyword arguments to pass to the vLLM engine.
278
+ For example, `tensor_parallel_size=2`.
383
279
  """
384
280
  ...
385
281
 
386
282
  @typing.overload
387
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
283
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
284
+ """
285
+ Specifies the PyPI packages for the step.
286
+
287
+ Information in this decorator will augment any
288
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
289
+ you can use `@pypi_base` to set packages required by all
290
+ steps and use `@pypi` to specify step-specific overrides.
291
+
292
+
293
+ Parameters
294
+ ----------
295
+ packages : Dict[str, str], default: {}
296
+ Packages to use for this step. The key is the name of the package
297
+ and the value is the version to use.
298
+ python : str, optional, default: None
299
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
300
+ that the version used will correspond to the version of the Python interpreter used to start the run.
301
+ """
388
302
  ...
389
303
 
390
304
  @typing.overload
391
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
305
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
392
306
  ...
393
307
 
394
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
308
+ @typing.overload
309
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
310
+ ...
311
+
312
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
395
313
  """
396
- Specifies the number of times the task corresponding
397
- to a step needs to be retried.
398
-
399
- This decorator is useful for handling transient errors, such as networking issues.
400
- If your task contains operations that can't be retried safely, e.g. database updates,
401
- it is advisable to annotate it with `@retry(times=0)`.
314
+ Specifies the PyPI packages for the step.
402
315
 
403
- This can be used in conjunction with the `@catch` decorator. The `@catch`
404
- decorator will execute a no-op task after all retries have been exhausted,
405
- ensuring that the flow execution can continue.
316
+ Information in this decorator will augment any
317
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
318
+ you can use `@pypi_base` to set packages required by all
319
+ steps and use `@pypi` to specify step-specific overrides.
406
320
 
407
321
 
408
322
  Parameters
409
323
  ----------
410
- times : int, default 3
411
- Number of times to retry this task.
412
- minutes_between_retries : int, default 2
413
- Number of minutes between retries.
324
+ packages : Dict[str, str], default: {}
325
+ Packages to use for this step. The key is the name of the package
326
+ and the value is the version to use.
327
+ python : str, optional, default: None
328
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
329
+ that the version used will correspond to the version of the Python interpreter used to start the run.
414
330
  """
415
331
  ...
416
332
 
417
333
  @typing.overload
418
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
334
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
419
335
  """
420
336
  Decorator prototype for all step decorators. This function gets specialized
421
337
  and imported for all decorators types by _import_plugin_decorators().
@@ -423,68 +339,19 @@ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.
423
339
  ...
424
340
 
425
341
  @typing.overload
426
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
342
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
427
343
  ...
428
344
 
429
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
345
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
430
346
  """
431
347
  Decorator prototype for all step decorators. This function gets specialized
432
348
  and imported for all decorators types by _import_plugin_decorators().
433
349
  """
434
350
  ...
435
351
 
436
- @typing.overload
437
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
352
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
438
353
  """
439
- Creates a human-readable report, a Metaflow Card, after this step completes.
440
-
441
- Note that you may add multiple `@card` decorators in a step with different parameters.
442
-
443
-
444
- Parameters
445
- ----------
446
- type : str, default 'default'
447
- Card type.
448
- id : str, optional, default None
449
- If multiple cards are present, use this id to identify this card.
450
- options : Dict[str, Any], default {}
451
- Options passed to the card. The contents depend on the card type.
452
- timeout : int, default 45
453
- Interrupt reporting if it takes more than this many seconds.
454
- """
455
- ...
456
-
457
- @typing.overload
458
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
459
- ...
460
-
461
- @typing.overload
462
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
463
- ...
464
-
465
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
466
- """
467
- Creates a human-readable report, a Metaflow Card, after this step completes.
468
-
469
- Note that you may add multiple `@card` decorators in a step with different parameters.
470
-
471
-
472
- Parameters
473
- ----------
474
- type : str, default 'default'
475
- Card type.
476
- id : str, optional, default None
477
- If multiple cards are present, use this id to identify this card.
478
- options : Dict[str, Any], default {}
479
- Options passed to the card. The contents depend on the card type.
480
- timeout : int, default 45
481
- Interrupt reporting if it takes more than this many seconds.
482
- """
483
- ...
484
-
485
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
486
- """
487
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
354
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
488
355
 
489
356
  User code call
490
357
  --------------
@@ -526,279 +393,103 @@ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy:
526
393
  ...
527
394
 
528
395
  @typing.overload
529
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
396
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
530
397
  """
531
- Enables checkpointing for a step.
532
-
533
- > Examples
534
-
535
- - Saving Checkpoints
536
-
537
- ```python
538
- @checkpoint
539
- @step
540
- def train(self):
541
- model = create_model(self.parameters, checkpoint_path = None)
542
- for i in range(self.epochs):
543
- # some training logic
544
- loss = model.train(self.dataset)
545
- if i % 10 == 0:
546
- model.save(
547
- current.checkpoint.directory,
548
- )
549
- # saves the contents of the `current.checkpoint.directory` as a checkpoint
550
- # and returns a reference dictionary to the checkpoint saved in the datastore
551
- self.latest_checkpoint = current.checkpoint.save(
552
- name="epoch_checkpoint",
553
- metadata={
554
- "epoch": i,
555
- "loss": loss,
556
- }
557
- )
558
- ```
559
-
560
- - Using Loaded Checkpoints
398
+ A simple decorator that demonstrates using CardDecoratorInjector
399
+ to inject a card and render simple markdown content.
400
+ """
401
+ ...
402
+
403
+ @typing.overload
404
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
405
+ ...
406
+
407
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
408
+ """
409
+ A simple decorator that demonstrates using CardDecoratorInjector
410
+ to inject a card and render simple markdown content.
411
+ """
412
+ ...
413
+
414
+ @typing.overload
415
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
416
+ """
417
+ Specifies the number of times the task corresponding
418
+ to a step needs to be retried.
561
419
 
562
- ```python
563
- @retry(times=3)
564
- @checkpoint
565
- @step
566
- def train(self):
567
- # Assume that the task has restarted and the previous attempt of the task
568
- # saved a checkpoint
569
- checkpoint_path = None
570
- if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
571
- print("Loaded checkpoint from the previous attempt")
572
- checkpoint_path = current.checkpoint.directory
420
+ This decorator is useful for handling transient errors, such as networking issues.
421
+ If your task contains operations that can't be retried safely, e.g. database updates,
422
+ it is advisable to annotate it with `@retry(times=0)`.
573
423
 
574
- model = create_model(self.parameters, checkpoint_path = checkpoint_path)
575
- for i in range(self.epochs):
576
- ...
577
- ```
424
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
425
+ decorator will execute a no-op task after all retries have been exhausted,
426
+ ensuring that the flow execution can continue.
578
427
 
579
428
 
580
429
  Parameters
581
430
  ----------
582
- load_policy : str, default: "fresh"
583
- The policy for loading the checkpoint. The following policies are supported:
584
- - "eager": Loads the the latest available checkpoint within the namespace.
585
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
586
- will be loaded at the start of the task.
587
- - "none": Do not load any checkpoint
588
- - "fresh": Loads the lastest checkpoint created within the running Task.
589
- This mode helps loading checkpoints across various retry attempts of the same task.
590
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
591
- created within the task will be loaded when the task is retries execution on failure.
592
-
593
- temp_dir_root : str, default: None
594
- The root directory under which `current.checkpoint.directory` will be created.
431
+ times : int, default 3
432
+ Number of times to retry this task.
433
+ minutes_between_retries : int, default 2
434
+ Number of minutes between retries.
595
435
  """
596
436
  ...
597
437
 
598
438
  @typing.overload
599
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
439
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
600
440
  ...
601
441
 
602
442
  @typing.overload
603
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
443
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
604
444
  ...
605
445
 
606
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
446
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
607
447
  """
608
- Enables checkpointing for a step.
609
-
610
- > Examples
611
-
612
- - Saving Checkpoints
613
-
614
- ```python
615
- @checkpoint
616
- @step
617
- def train(self):
618
- model = create_model(self.parameters, checkpoint_path = None)
619
- for i in range(self.epochs):
620
- # some training logic
621
- loss = model.train(self.dataset)
622
- if i % 10 == 0:
623
- model.save(
624
- current.checkpoint.directory,
625
- )
626
- # saves the contents of the `current.checkpoint.directory` as a checkpoint
627
- # and returns a reference dictionary to the checkpoint saved in the datastore
628
- self.latest_checkpoint = current.checkpoint.save(
629
- name="epoch_checkpoint",
630
- metadata={
631
- "epoch": i,
632
- "loss": loss,
633
- }
634
- )
635
- ```
636
-
637
- - Using Loaded Checkpoints
448
+ Specifies the number of times the task corresponding
449
+ to a step needs to be retried.
638
450
 
639
- ```python
640
- @retry(times=3)
641
- @checkpoint
642
- @step
643
- def train(self):
644
- # Assume that the task has restarted and the previous attempt of the task
645
- # saved a checkpoint
646
- checkpoint_path = None
647
- if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
648
- print("Loaded checkpoint from the previous attempt")
649
- checkpoint_path = current.checkpoint.directory
451
+ This decorator is useful for handling transient errors, such as networking issues.
452
+ If your task contains operations that can't be retried safely, e.g. database updates,
453
+ it is advisable to annotate it with `@retry(times=0)`.
650
454
 
651
- model = create_model(self.parameters, checkpoint_path = checkpoint_path)
652
- for i in range(self.epochs):
653
- ...
654
- ```
455
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
456
+ decorator will execute a no-op task after all retries have been exhausted,
457
+ ensuring that the flow execution can continue.
655
458
 
656
459
 
657
460
  Parameters
658
461
  ----------
659
- load_policy : str, default: "fresh"
660
- The policy for loading the checkpoint. The following policies are supported:
661
- - "eager": Loads the the latest available checkpoint within the namespace.
662
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
663
- will be loaded at the start of the task.
664
- - "none": Do not load any checkpoint
665
- - "fresh": Loads the lastest checkpoint created within the running Task.
666
- This mode helps loading checkpoints across various retry attempts of the same task.
667
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
668
- created within the task will be loaded when the task is retries execution on failure.
669
-
670
- temp_dir_root : str, default: None
671
- The root directory under which `current.checkpoint.directory` will be created.
462
+ times : int, default 3
463
+ Number of times to retry this task.
464
+ minutes_between_retries : int, default 2
465
+ Number of minutes between retries.
672
466
  """
673
467
  ...
674
468
 
675
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
469
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
676
470
  """
677
- Specifies that this step should execute on DGX cloud.
471
+ Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
472
+ for S3 read and write requests.
678
473
 
474
+ This decorator requires an integration in the Outerbounds platform that
475
+ points to an external bucket. It affects S3 operations performed via
476
+ Metaflow's `get_aws_client` and `S3` within a `@step`.
679
477
 
680
- Parameters
681
- ----------
682
- gpu : int
683
- Number of GPUs to use.
684
- gpu_type : str
685
- Type of Nvidia GPU to use.
686
- """
687
- ...
688
-
689
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
690
- """
691
- Specifies that this step should execute on Kubernetes.
478
+ Read operations
479
+ ---------------
480
+ All read operations pass through the proxy. If an object does not already
481
+ exist in the external bucket, it is cached there. For example, if code reads
482
+ from buckets `FOO` and `BAR` using the `S3` interface, objects from both
483
+ buckets are cached in the external bucket.
692
484
 
693
-
694
- Parameters
695
- ----------
696
- cpu : int, default 1
697
- Number of CPUs required for this step. If `@resources` is
698
- also present, the maximum value from all decorators is used.
699
- memory : int, default 4096
700
- Memory size (in MB) required for this step. If
701
- `@resources` is also present, the maximum value from all decorators is
702
- used.
703
- disk : int, default 10240
704
- Disk size (in MB) required for this step. If
705
- `@resources` is also present, the maximum value from all decorators is
706
- used.
707
- image : str, optional, default None
708
- Docker image to use when launching on Kubernetes. If not specified, and
709
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
710
- not, a default Docker image mapping to the current version of Python is used.
711
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
712
- If given, the imagePullPolicy to be applied to the Docker image of the step.
713
- image_pull_secrets: List[str], default []
714
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
715
- Kubernetes image pull secrets to use when pulling container images
716
- in Kubernetes.
717
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
718
- Kubernetes service account to use when launching pod in Kubernetes.
719
- secrets : List[str], optional, default None
720
- Kubernetes secrets to use when launching pod in Kubernetes. These
721
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
722
- in Metaflow configuration.
723
- node_selector: Union[Dict[str,str], str], optional, default None
724
- Kubernetes node selector(s) to apply to the pod running the task.
725
- Can be passed in as a comma separated string of values e.g.
726
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
727
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
728
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
729
- Kubernetes namespace to use when launching pod in Kubernetes.
730
- gpu : int, optional, default None
731
- Number of GPUs required for this step. A value of zero implies that
732
- the scheduled node should not have GPUs.
733
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
734
- The vendor of the GPUs to be used for this step.
735
- tolerations : List[Dict[str,str]], default []
736
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
737
- Kubernetes tolerations to use when launching pod in Kubernetes.
738
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
739
- Kubernetes labels to use when launching pod in Kubernetes.
740
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
741
- Kubernetes annotations to use when launching pod in Kubernetes.
742
- use_tmpfs : bool, default False
743
- This enables an explicit tmpfs mount for this step.
744
- tmpfs_tempdir : bool, default True
745
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
746
- tmpfs_size : int, optional, default: None
747
- The value for the size (in MiB) of the tmpfs mount for this step.
748
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
749
- memory allocated for this step.
750
- tmpfs_path : str, optional, default /metaflow_temp
751
- Path to tmpfs mount for this step.
752
- persistent_volume_claims : Dict[str, str], optional, default None
753
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
754
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
755
- shared_memory: int, optional
756
- Shared memory size (in MiB) required for this step
757
- port: int, optional
758
- Port number to specify in the Kubernetes job object
759
- compute_pool : str, optional, default None
760
- Compute pool to be used for for this step.
761
- If not specified, any accessible compute pool within the perimeter is used.
762
- hostname_resolution_timeout: int, default 10 * 60
763
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
764
- Only applicable when @parallel is used.
765
- qos: str, default: Burstable
766
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
767
-
768
- security_context: Dict[str, Any], optional, default None
769
- Container security context. Applies to the task container. Allows the following keys:
770
- - privileged: bool, optional, default None
771
- - allow_privilege_escalation: bool, optional, default None
772
- - run_as_user: int, optional, default None
773
- - run_as_group: int, optional, default None
774
- - run_as_non_root: bool, optional, default None
775
- """
776
- ...
777
-
778
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
779
- """
780
- Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
781
- for S3 read and write requests.
782
-
783
- This decorator requires an integration in the Outerbounds platform that
784
- points to an external bucket. It affects S3 operations performed via
785
- Metaflow's `get_aws_client` and `S3` within a `@step`.
786
-
787
- Read operations
788
- ---------------
789
- All read operations pass through the proxy. If an object does not already
790
- exist in the external bucket, it is cached there. For example, if code reads
791
- from buckets `FOO` and `BAR` using the `S3` interface, objects from both
792
- buckets are cached in the external bucket.
793
-
794
- During task execution, all S3‑related read requests are routed through the
795
- proxy:
796
- - If the object is present in the external object store, the proxy
797
- streams it directly from there without accessing the requested origin
798
- bucket.
799
- - If the object is not present in the external storage, the proxy
800
- fetches it from the requested bucket, caches it in the external
801
- storage, and streams the response from the origin bucket.
485
+ During task execution, all S3‑related read requests are routed through the
486
+ proxy:
487
+ - If the object is present in the external object store, the proxy
488
+ streams it directly from there without accessing the requested origin
489
+ bucket.
490
+ - If the object is not present in the external storage, the proxy
491
+ fetches it from the requested bucket, caches it in the external
492
+ storage, and streams the response from the origin bucket.
802
493
 
803
494
  Warning
804
495
  -------
@@ -834,120 +525,169 @@ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typin
834
525
  """
835
526
  ...
836
527
 
837
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
528
+ @typing.overload
529
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
838
530
  """
839
- Decorator that helps cache, version, and store models/datasets from the Hugging Face Hub.
840
-
841
- Examples
842
- --------
531
+ Internal decorator to support Fast bakery
532
+ """
533
+ ...
534
+
535
+ @typing.overload
536
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
537
+ ...
538
+
539
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
540
+ """
541
+ Internal decorator to support Fast bakery
542
+ """
543
+ ...
544
+
545
+ @typing.overload
546
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
547
+ """
548
+ Specifies a timeout for your step.
843
549
 
844
- ```python
845
- # **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
846
- @huggingface_hub
847
- @step
848
- def pull_model_from_huggingface(self):
849
- # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
850
- # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
851
- # with the same `repo_id` in the backend storage, it will not download the model again. The return
852
- # value of the function is a reference to the model in the backend storage.
853
- # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
550
+ This decorator is useful if this step may hang indefinitely.
854
551
 
855
- self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
856
- self.llama_model = current.huggingface_hub.snapshot_download(
857
- repo_id=self.model_id,
858
- allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
859
- )
860
- self.next(self.train)
552
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
553
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
554
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
861
555
 
862
- # **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
863
- @huggingface_hub
864
- @step
865
- def run_training(self):
866
- # Temporary directory (auto-cleaned on exit)
867
- with current.huggingface_hub.load(
868
- repo_id="google-bert/bert-base-uncased",
869
- allow_patterns=["*.bin"],
870
- ) as local_path:
871
- # Use files under local_path
872
- train_model(local_path)
873
- ...
556
+ Note that all the values specified in parameters are added together so if you specify
557
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
874
558
 
875
- # **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
876
559
 
877
- @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
878
- @step
879
- def pull_model_from_huggingface(self):
880
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
560
+ Parameters
561
+ ----------
562
+ seconds : int, default 0
563
+ Number of seconds to wait prior to timing out.
564
+ minutes : int, default 0
565
+ Number of minutes to wait prior to timing out.
566
+ hours : int, default 0
567
+ Number of hours to wait prior to timing out.
568
+ """
569
+ ...
570
+
571
+ @typing.overload
572
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
573
+ ...
574
+
575
+ @typing.overload
576
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
577
+ ...
578
+
579
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
580
+ """
581
+ Specifies a timeout for your step.
881
582
 
882
- @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
883
- @step
884
- def finetune_model(self):
885
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
886
- # path_to_model will be /my-directory
583
+ This decorator is useful if this step may hang indefinitely.
887
584
 
585
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
586
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
587
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
888
588
 
889
- # Takes all the arguments passed to `snapshot_download`
890
- # except for `local_dir`
891
- @huggingface_hub(load=[
892
- {
893
- "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
894
- },
895
- {
896
- "repo_id": "myorg/mistral-lora",
897
- "repo_type": "model",
898
- },
899
- ])
900
- @step
901
- def finetune_model(self):
902
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
903
- # path_to_model will be /my-directory
904
- ```
589
+ Note that all the values specified in parameters are added together so if you specify
590
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
905
591
 
906
592
 
907
593
  Parameters
908
594
  ----------
909
- temp_dir_root : str, optional
910
- The root directory that will hold the temporary directory where objects will be downloaded.
595
+ seconds : int, default 0
596
+ Number of seconds to wait prior to timing out.
597
+ minutes : int, default 0
598
+ Number of minutes to wait prior to timing out.
599
+ hours : int, default 0
600
+ Number of hours to wait prior to timing out.
601
+ """
602
+ ...
603
+
604
+ @typing.overload
605
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
606
+ """
607
+ Specifies that the step will success under all circumstances.
911
608
 
912
- cache_scope : str, optional
913
- The scope of the cache. Can be `checkpoint` / `flow` / `global`.
914
- - `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
915
- i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
916
- Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
609
+ The decorator will create an optional artifact, specified by `var`, which
610
+ contains the exception raised. You can use it to detect the presence
611
+ of errors, indicating that all happy-path artifacts produced by the step
612
+ are missing.
917
613
 
918
- - `flow`: All repos are cached under the flow, regardless of namespace.
919
- i.e., the cached path is derived solely from the flow name.
920
- When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
921
614
 
922
- - `global`: All repos are cached under a globally static path.
923
- i.e., the base path of the cache is static and all repos are stored under it.
924
- When to use this mode:
925
- - All repos from the Hugging Face Hub need to be shared by users across all flow executions.
926
- - Each caching scope comes with its own trade-offs:
927
- - `checkpoint`:
928
- - Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
929
- - Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
930
- - `flow`:
931
- - Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
932
- - The blast radius of a bad checkpoint is limited to all runs of a particular flow.
933
- - It doesn't promote cache reuse across flows.
934
- - `global`:
935
- - Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
936
- - It promotes cache reuse across flows.
937
- - The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
615
+ Parameters
616
+ ----------
617
+ var : str, optional, default None
618
+ Name of the artifact in which to store the caught exception.
619
+ If not specified, the exception is not stored.
620
+ print_exception : bool, default True
621
+ Determines whether or not the exception is printed to
622
+ stdout when caught.
623
+ """
624
+ ...
625
+
626
+ @typing.overload
627
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
628
+ ...
629
+
630
+ @typing.overload
631
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
632
+ ...
633
+
634
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
635
+ """
636
+ Specifies that the step will success under all circumstances.
938
637
 
939
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
940
- The list of repos (models/datasets) to load.
638
+ The decorator will create an optional artifact, specified by `var`, which
639
+ contains the exception raised. You can use it to detect the presence
640
+ of errors, indicating that all happy-path artifacts produced by the step
641
+ are missing.
941
642
 
942
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
943
643
 
944
- - If repo (model/dataset) is not found in the datastore:
945
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
946
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
947
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
644
+ Parameters
645
+ ----------
646
+ var : str, optional, default None
647
+ Name of the artifact in which to store the caught exception.
648
+ If not specified, the exception is not stored.
649
+ print_exception : bool, default True
650
+ Determines whether or not the exception is printed to
651
+ stdout when caught.
652
+ """
653
+ ...
654
+
655
+ @typing.overload
656
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
657
+ """
658
+ Specifies secrets to be retrieved and injected as environment variables prior to
659
+ the execution of a step.
948
660
 
949
- - If repo is found in the datastore:
950
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
661
+
662
+ Parameters
663
+ ----------
664
+ sources : List[Union[str, Dict[str, Any]]], default: []
665
+ List of secret specs, defining how the secrets are to be retrieved
666
+ role : str, optional, default: None
667
+ Role to use for fetching secrets
668
+ """
669
+ ...
670
+
671
+ @typing.overload
672
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
673
+ ...
674
+
675
+ @typing.overload
676
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
677
+ ...
678
+
679
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
680
+ """
681
+ Specifies secrets to be retrieved and injected as environment variables prior to
682
+ the execution of a step.
683
+
684
+
685
+ Parameters
686
+ ----------
687
+ sources : List[Union[str, Dict[str, Any]]], default: []
688
+ List of secret specs, defining how the secrets are to be retrieved
689
+ role : str, optional, default: None
690
+ Role to use for fetching secrets
951
691
  """
952
692
  ...
953
693
 
@@ -1096,310 +836,622 @@ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[t
1096
836
  """
1097
837
  ...
1098
838
 
1099
- def coreweave_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
839
+ @typing.overload
840
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1100
841
  """
1101
- `@coreweave_s3_proxy` is a CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1102
- It exists to make it easier for users to know that this decorator should only be used with
1103
- a Neo Cloud like CoreWeave. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
842
+ Specifies environment variables to be set prior to the execution of a step.
1104
843
 
1105
844
 
1106
- Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
1107
- for S3 read and write requests.
845
+ Parameters
846
+ ----------
847
+ vars : Dict[str, str], default {}
848
+ Dictionary of environment variables to set.
849
+ """
850
+ ...
851
+
852
+ @typing.overload
853
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
854
+ ...
855
+
856
+ @typing.overload
857
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
858
+ ...
859
+
860
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
861
+ """
862
+ Specifies environment variables to be set prior to the execution of a step.
1108
863
 
1109
- This decorator requires an integration in the Outerbounds platform that
1110
- points to an external bucket. It affects S3 operations performed via
1111
- Metaflow's `get_aws_client` and `S3` within a `@step`.
1112
864
 
1113
- Read operations
1114
- ---------------
1115
- All read operations pass through the proxy. If an object does not already
1116
- exist in the external bucket, it is cached there. For example, if code reads
1117
- from buckets `FOO` and `BAR` using the `S3` interface, objects from both
1118
- buckets are cached in the external bucket.
865
+ Parameters
866
+ ----------
867
+ vars : Dict[str, str], default {}
868
+ Dictionary of environment variables to set.
869
+ """
870
+ ...
871
+
872
+ @typing.overload
873
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
874
+ """
875
+ Decorator prototype for all step decorators. This function gets specialized
876
+ and imported for all decorators types by _import_plugin_decorators().
877
+ """
878
+ ...
879
+
880
+ @typing.overload
881
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
882
+ ...
883
+
884
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
885
+ """
886
+ Decorator prototype for all step decorators. This function gets specialized
887
+ and imported for all decorators types by _import_plugin_decorators().
888
+ """
889
+ ...
890
+
891
+ @typing.overload
892
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
893
+ """
894
+ Enables checkpointing for a step.
1119
895
 
1120
- During task execution, all S3‑related read requests are routed through the
1121
- proxy:
1122
- - If the object is present in the external object store, the proxy
1123
- streams it directly from there without accessing the requested origin
1124
- bucket.
1125
- - If the object is not present in the external storage, the proxy
1126
- fetches it from the requested bucket, caches it in the external
1127
- storage, and streams the response from the origin bucket.
896
+ > Examples
1128
897
 
1129
- Warning
1130
- -------
1131
- All READ operations (e.g., GetObject, HeadObject) pass through the external
1132
- bucket regardless of the bucket specified in user code. Even
1133
- `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
1134
- external bucket cache.
898
+ - Saving Checkpoints
1135
899
 
1136
- Write operations
1137
- ----------------
1138
- Write behavior is controlled by the `write_mode` parameter, which determines
1139
- whether writes also persist objects in the cache.
900
+ ```python
901
+ @checkpoint
902
+ @step
903
+ def train(self):
904
+ model = create_model(self.parameters, checkpoint_path = None)
905
+ for i in range(self.epochs):
906
+ # some training logic
907
+ loss = model.train(self.dataset)
908
+ if i % 10 == 0:
909
+ model.save(
910
+ current.checkpoint.directory,
911
+ )
912
+ # saves the contents of the `current.checkpoint.directory` as a checkpoint
913
+ # and returns a reference dictionary to the checkpoint saved in the datastore
914
+ self.latest_checkpoint = current.checkpoint.save(
915
+ name="epoch_checkpoint",
916
+ metadata={
917
+ "epoch": i,
918
+ "loss": loss,
919
+ }
920
+ )
921
+ ```
1140
922
 
1141
- `write_mode` values:
1142
- - `origin-and-cache`: objects are written both to the cache and to their
1143
- intended origin bucket.
1144
- - `origin`: objects are written only to their intended origin bucket.
923
+ - Using Loaded Checkpoints
924
+
925
+ ```python
926
+ @retry(times=3)
927
+ @checkpoint
928
+ @step
929
+ def train(self):
930
+ # Assume that the task has restarted and the previous attempt of the task
931
+ # saved a checkpoint
932
+ checkpoint_path = None
933
+ if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
934
+ print("Loaded checkpoint from the previous attempt")
935
+ checkpoint_path = current.checkpoint.directory
936
+
937
+ model = create_model(self.parameters, checkpoint_path = checkpoint_path)
938
+ for i in range(self.epochs):
939
+ ...
940
+ ```
1145
941
 
1146
942
 
1147
943
  Parameters
1148
944
  ----------
1149
- integration_name : str, optional
1150
- [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
1151
- that holds the configuration for the external, S3‑compatible object
1152
- storage bucket. If not specified, the only available S3 proxy
1153
- integration in the namespace is used (fails if multiple exist).
1154
- write_mode : str, optional
1155
- Controls whether writes also go to the external bucket.
1156
- - `origin` (default)
1157
- - `origin-and-cache`
1158
- debug : bool, optional
1159
- Enables debug logging for proxy operations.
945
+ load_policy : str, default: "fresh"
946
+ The policy for loading the checkpoint. The following policies are supported:
947
+ - "eager": Loads the the latest available checkpoint within the namespace.
948
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
949
+ will be loaded at the start of the task.
950
+ - "none": Do not load any checkpoint
951
+ - "fresh": Loads the lastest checkpoint created within the running Task.
952
+ This mode helps loading checkpoints across various retry attempts of the same task.
953
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
954
+ created within the task will be loaded when the task is retries execution on failure.
955
+
956
+ temp_dir_root : str, default: None
957
+ The root directory under which `current.checkpoint.directory` will be created.
1160
958
  """
1161
959
  ...
1162
960
 
1163
961
  @typing.overload
1164
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
962
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
963
+ ...
964
+
965
+ @typing.overload
966
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
967
+ ...
968
+
969
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
1165
970
  """
1166
- Specifies a timeout for your step.
971
+ Enables checkpointing for a step.
1167
972
 
1168
- This decorator is useful if this step may hang indefinitely.
973
+ > Examples
1169
974
 
1170
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1171
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1172
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
975
+ - Saving Checkpoints
1173
976
 
1174
- Note that all the values specified in parameters are added together so if you specify
1175
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
977
+ ```python
978
+ @checkpoint
979
+ @step
980
+ def train(self):
981
+ model = create_model(self.parameters, checkpoint_path = None)
982
+ for i in range(self.epochs):
983
+ # some training logic
984
+ loss = model.train(self.dataset)
985
+ if i % 10 == 0:
986
+ model.save(
987
+ current.checkpoint.directory,
988
+ )
989
+ # saves the contents of the `current.checkpoint.directory` as a checkpoint
990
+ # and returns a reference dictionary to the checkpoint saved in the datastore
991
+ self.latest_checkpoint = current.checkpoint.save(
992
+ name="epoch_checkpoint",
993
+ metadata={
994
+ "epoch": i,
995
+ "loss": loss,
996
+ }
997
+ )
998
+ ```
999
+
1000
+ - Using Loaded Checkpoints
1001
+
1002
+ ```python
1003
+ @retry(times=3)
1004
+ @checkpoint
1005
+ @step
1006
+ def train(self):
1007
+ # Assume that the task has restarted and the previous attempt of the task
1008
+ # saved a checkpoint
1009
+ checkpoint_path = None
1010
+ if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
1011
+ print("Loaded checkpoint from the previous attempt")
1012
+ checkpoint_path = current.checkpoint.directory
1013
+
1014
+ model = create_model(self.parameters, checkpoint_path = checkpoint_path)
1015
+ for i in range(self.epochs):
1016
+ ...
1017
+ ```
1176
1018
 
1177
1019
 
1178
1020
  Parameters
1179
1021
  ----------
1180
- seconds : int, default 0
1181
- Number of seconds to wait prior to timing out.
1182
- minutes : int, default 0
1183
- Number of minutes to wait prior to timing out.
1184
- hours : int, default 0
1185
- Number of hours to wait prior to timing out.
1022
+ load_policy : str, default: "fresh"
1023
+ The policy for loading the checkpoint. The following policies are supported:
1024
+ - "eager": Loads the the latest available checkpoint within the namespace.
1025
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
1026
+ will be loaded at the start of the task.
1027
+ - "none": Do not load any checkpoint
1028
+ - "fresh": Loads the lastest checkpoint created within the running Task.
1029
+ This mode helps loading checkpoints across various retry attempts of the same task.
1030
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
1031
+ created within the task will be loaded when the task is retries execution on failure.
1032
+
1033
+ temp_dir_root : str, default: None
1034
+ The root directory under which `current.checkpoint.directory` will be created.
1186
1035
  """
1187
1036
  ...
1188
1037
 
1189
- @typing.overload
1190
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1191
- ...
1192
-
1193
- @typing.overload
1194
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1038
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1039
+ """
1040
+ Specifies that this step should execute on Kubernetes.
1041
+
1042
+
1043
+ Parameters
1044
+ ----------
1045
+ cpu : int, default 1
1046
+ Number of CPUs required for this step. If `@resources` is
1047
+ also present, the maximum value from all decorators is used.
1048
+ memory : int, default 4096
1049
+ Memory size (in MB) required for this step. If
1050
+ `@resources` is also present, the maximum value from all decorators is
1051
+ used.
1052
+ disk : int, default 10240
1053
+ Disk size (in MB) required for this step. If
1054
+ `@resources` is also present, the maximum value from all decorators is
1055
+ used.
1056
+ image : str, optional, default None
1057
+ Docker image to use when launching on Kubernetes. If not specified, and
1058
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1059
+ not, a default Docker image mapping to the current version of Python is used.
1060
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1061
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
1062
+ image_pull_secrets: List[str], default []
1063
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
1064
+ Kubernetes image pull secrets to use when pulling container images
1065
+ in Kubernetes.
1066
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1067
+ Kubernetes service account to use when launching pod in Kubernetes.
1068
+ secrets : List[str], optional, default None
1069
+ Kubernetes secrets to use when launching pod in Kubernetes. These
1070
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1071
+ in Metaflow configuration.
1072
+ node_selector: Union[Dict[str,str], str], optional, default None
1073
+ Kubernetes node selector(s) to apply to the pod running the task.
1074
+ Can be passed in as a comma separated string of values e.g.
1075
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
1076
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
1077
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1078
+ Kubernetes namespace to use when launching pod in Kubernetes.
1079
+ gpu : int, optional, default None
1080
+ Number of GPUs required for this step. A value of zero implies that
1081
+ the scheduled node should not have GPUs.
1082
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1083
+ The vendor of the GPUs to be used for this step.
1084
+ tolerations : List[Dict[str,str]], default []
1085
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1086
+ Kubernetes tolerations to use when launching pod in Kubernetes.
1087
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
1088
+ Kubernetes labels to use when launching pod in Kubernetes.
1089
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
1090
+ Kubernetes annotations to use when launching pod in Kubernetes.
1091
+ use_tmpfs : bool, default False
1092
+ This enables an explicit tmpfs mount for this step.
1093
+ tmpfs_tempdir : bool, default True
1094
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1095
+ tmpfs_size : int, optional, default: None
1096
+ The value for the size (in MiB) of the tmpfs mount for this step.
1097
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1098
+ memory allocated for this step.
1099
+ tmpfs_path : str, optional, default /metaflow_temp
1100
+ Path to tmpfs mount for this step.
1101
+ persistent_volume_claims : Dict[str, str], optional, default None
1102
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1103
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1104
+ shared_memory: int, optional
1105
+ Shared memory size (in MiB) required for this step
1106
+ port: int, optional
1107
+ Port number to specify in the Kubernetes job object
1108
+ compute_pool : str, optional, default None
1109
+ Compute pool to be used for for this step.
1110
+ If not specified, any accessible compute pool within the perimeter is used.
1111
+ hostname_resolution_timeout: int, default 10 * 60
1112
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
1113
+ Only applicable when @parallel is used.
1114
+ qos: str, default: Burstable
1115
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
1116
+
1117
+ security_context: Dict[str, Any], optional, default None
1118
+ Container security context. Applies to the task container. Allows the following keys:
1119
+ - privileged: bool, optional, default None
1120
+ - allow_privilege_escalation: bool, optional, default None
1121
+ - run_as_user: int, optional, default None
1122
+ - run_as_group: int, optional, default None
1123
+ - run_as_non_root: bool, optional, default None
1124
+ """
1195
1125
  ...
1196
1126
 
1197
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1127
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1198
1128
  """
1199
- Specifies a timeout for your step.
1129
+ Decorator that helps cache, version, and store models/datasets from the Hugging Face Hub.
1200
1130
 
1201
- This decorator is useful if this step may hang indefinitely.
1131
+ Examples
1132
+ --------
1133
+
1134
+ ```python
1135
+ # **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
1136
+ @huggingface_hub
1137
+ @step
1138
+ def pull_model_from_huggingface(self):
1139
+ # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
1140
+ # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
1141
+ # with the same `repo_id` in the backend storage, it will not download the model again. The return
1142
+ # value of the function is a reference to the model in the backend storage.
1143
+ # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
1144
+
1145
+ self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
1146
+ self.llama_model = current.huggingface_hub.snapshot_download(
1147
+ repo_id=self.model_id,
1148
+ allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
1149
+ )
1150
+ self.next(self.train)
1151
+
1152
+ # **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
1153
+ @huggingface_hub
1154
+ @step
1155
+ def run_training(self):
1156
+ # Temporary directory (auto-cleaned on exit)
1157
+ with current.huggingface_hub.load(
1158
+ repo_id="google-bert/bert-base-uncased",
1159
+ allow_patterns=["*.bin"],
1160
+ ) as local_path:
1161
+ # Use files under local_path
1162
+ train_model(local_path)
1163
+ ...
1164
+
1165
+ # **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
1166
+
1167
+ @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
1168
+ @step
1169
+ def pull_model_from_huggingface(self):
1170
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
1171
+
1172
+ @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
1173
+ @step
1174
+ def finetune_model(self):
1175
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
1176
+ # path_to_model will be /my-directory
1177
+
1178
+
1179
+ # Takes all the arguments passed to `snapshot_download`
1180
+ # except for `local_dir`
1181
+ @huggingface_hub(load=[
1182
+ {
1183
+ "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
1184
+ },
1185
+ {
1186
+ "repo_id": "myorg/mistral-lora",
1187
+ "repo_type": "model",
1188
+ },
1189
+ ])
1190
+ @step
1191
+ def finetune_model(self):
1192
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
1193
+ # path_to_model will be /my-directory
1194
+ ```
1195
+
1196
+
1197
+ Parameters
1198
+ ----------
1199
+ temp_dir_root : str, optional
1200
+ The root directory that will hold the temporary directory where objects will be downloaded.
1201
+
1202
+ cache_scope : str, optional
1203
+ The scope of the cache. Can be `checkpoint` / `flow` / `global`.
1204
+ - `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
1205
+ i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
1206
+ Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
1207
+
1208
+ - `flow`: All repos are cached under the flow, regardless of namespace.
1209
+ i.e., the cached path is derived solely from the flow name.
1210
+ When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
1211
+
1212
+ - `global`: All repos are cached under a globally static path.
1213
+ i.e., the base path of the cache is static and all repos are stored under it.
1214
+ When to use this mode:
1215
+ - All repos from the Hugging Face Hub need to be shared by users across all flow executions.
1216
+ - Each caching scope comes with its own trade-offs:
1217
+ - `checkpoint`:
1218
+ - Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
1219
+ - Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
1220
+ - `flow`:
1221
+ - Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
1222
+ - The blast radius of a bad checkpoint is limited to all runs of a particular flow.
1223
+ - It doesn't promote cache reuse across flows.
1224
+ - `global`:
1225
+ - Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
1226
+ - It promotes cache reuse across flows.
1227
+ - The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
1202
1228
 
1203
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1204
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1205
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1229
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
1230
+ The list of repos (models/datasets) to load.
1206
1231
 
1207
- Note that all the values specified in parameters are added together so if you specify
1208
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1232
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
1209
1233
 
1234
+ - If repo (model/dataset) is not found in the datastore:
1235
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
1236
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
1237
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
1210
1238
 
1211
- Parameters
1212
- ----------
1213
- seconds : int, default 0
1214
- Number of seconds to wait prior to timing out.
1215
- minutes : int, default 0
1216
- Number of minutes to wait prior to timing out.
1217
- hours : int, default 0
1218
- Number of hours to wait prior to timing out.
1239
+ - If repo is found in the datastore:
1240
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
1219
1241
  """
1220
1242
  ...
1221
1243
 
1222
- def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1244
+ @typing.overload
1245
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1223
1246
  """
1224
- This decorator is used to run vllm APIs as Metaflow task sidecars.
1225
-
1226
- User code call
1227
- --------------
1228
- @vllm(
1229
- model="...",
1230
- ...
1231
- )
1232
-
1233
- Valid backend options
1234
- ---------------------
1235
- - 'local': Run as a separate process on the local task machine.
1236
-
1237
- Valid model options
1238
- -------------------
1239
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
1247
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1240
1248
 
1241
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
1242
- If you need multiple models, you must create multiple @vllm decorators.
1249
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1243
1250
 
1244
1251
 
1245
1252
  Parameters
1246
1253
  ----------
1247
- model: str
1248
- HuggingFace model identifier to be served by vLLM.
1249
- backend: str
1250
- Determines where and how to run the vLLM process.
1251
- openai_api_server: bool
1252
- Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
1253
- Default is False (uses native engine).
1254
- Set to True for backward compatibility with existing code.
1255
- debug: bool
1256
- Whether to turn on verbose debugging logs.
1257
- card_refresh_interval: int
1258
- Interval in seconds for refreshing the vLLM status card.
1259
- Only used when openai_api_server=True.
1260
- max_retries: int
1261
- Maximum number of retries checking for vLLM server startup.
1262
- Only used when openai_api_server=True.
1263
- retry_alert_frequency: int
1264
- Frequency of alert logs for vLLM server startup retries.
1265
- Only used when openai_api_server=True.
1266
- engine_args : dict
1267
- Additional keyword arguments to pass to the vLLM engine.
1268
- For example, `tensor_parallel_size=2`.
1254
+ type : str, default 'default'
1255
+ Card type.
1256
+ id : str, optional, default None
1257
+ If multiple cards are present, use this id to identify this card.
1258
+ options : Dict[str, Any], default {}
1259
+ Options passed to the card. The contents depend on the card type.
1260
+ timeout : int, default 45
1261
+ Interrupt reporting if it takes more than this many seconds.
1269
1262
  """
1270
1263
  ...
1271
1264
 
1272
1265
  @typing.overload
1273
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1274
- """
1275
- Internal decorator to support Fast bakery
1276
- """
1266
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1277
1267
  ...
1278
1268
 
1279
1269
  @typing.overload
1280
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1281
- ...
1282
-
1283
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1284
- """
1285
- Internal decorator to support Fast bakery
1286
- """
1270
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1287
1271
  ...
1288
1272
 
1289
- @typing.overload
1290
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1273
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1291
1274
  """
1292
- Specifies secrets to be retrieved and injected as environment variables prior to
1293
- the execution of a step.
1275
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1276
+
1277
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1294
1278
 
1295
1279
 
1296
1280
  Parameters
1297
1281
  ----------
1298
- sources : List[Union[str, Dict[str, Any]]], default: []
1299
- List of secret specs, defining how the secrets are to be retrieved
1300
- role : str, optional, default: None
1301
- Role to use for fetching secrets
1282
+ type : str, default 'default'
1283
+ Card type.
1284
+ id : str, optional, default None
1285
+ If multiple cards are present, use this id to identify this card.
1286
+ options : Dict[str, Any], default {}
1287
+ Options passed to the card. The contents depend on the card type.
1288
+ timeout : int, default 45
1289
+ Interrupt reporting if it takes more than this many seconds.
1302
1290
  """
1303
1291
  ...
1304
1292
 
1305
- @typing.overload
1306
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1307
- ...
1308
-
1309
- @typing.overload
1310
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1311
- ...
1312
-
1313
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
1293
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1314
1294
  """
1315
- Specifies secrets to be retrieved and injected as environment variables prior to
1316
- the execution of a step.
1295
+ Specifies that this step should execute on DGX cloud.
1317
1296
 
1318
1297
 
1319
1298
  Parameters
1320
1299
  ----------
1321
- sources : List[Union[str, Dict[str, Any]]], default: []
1322
- List of secret specs, defining how the secrets are to be retrieved
1323
- role : str, optional, default: None
1324
- Role to use for fetching secrets
1300
+ gpu : int
1301
+ Number of GPUs to use.
1302
+ gpu_type : str
1303
+ Type of Nvidia GPU to use.
1325
1304
  """
1326
1305
  ...
1327
1306
 
1328
1307
  @typing.overload
1329
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1308
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1330
1309
  """
1331
- Specifies that the step will success under all circumstances.
1310
+ Specifies the resources needed when executing this step.
1332
1311
 
1333
- The decorator will create an optional artifact, specified by `var`, which
1334
- contains the exception raised. You can use it to detect the presence
1335
- of errors, indicating that all happy-path artifacts produced by the step
1336
- are missing.
1312
+ Use `@resources` to specify the resource requirements
1313
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1314
+
1315
+ You can choose the compute layer on the command line by executing e.g.
1316
+ ```
1317
+ python myflow.py run --with batch
1318
+ ```
1319
+ or
1320
+ ```
1321
+ python myflow.py run --with kubernetes
1322
+ ```
1323
+ which executes the flow on the desired system using the
1324
+ requirements specified in `@resources`.
1337
1325
 
1338
1326
 
1339
1327
  Parameters
1340
1328
  ----------
1341
- var : str, optional, default None
1342
- Name of the artifact in which to store the caught exception.
1343
- If not specified, the exception is not stored.
1344
- print_exception : bool, default True
1345
- Determines whether or not the exception is printed to
1346
- stdout when caught.
1329
+ cpu : int, default 1
1330
+ Number of CPUs required for this step.
1331
+ gpu : int, optional, default None
1332
+ Number of GPUs required for this step.
1333
+ disk : int, optional, default None
1334
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1335
+ memory : int, default 4096
1336
+ Memory size (in MB) required for this step.
1337
+ shared_memory : int, optional, default None
1338
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1339
+ This parameter maps to the `--shm-size` option in Docker.
1347
1340
  """
1348
1341
  ...
1349
1342
 
1350
1343
  @typing.overload
1351
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1344
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1352
1345
  ...
1353
1346
 
1354
1347
  @typing.overload
1355
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1348
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1356
1349
  ...
1357
1350
 
1358
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1351
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1359
1352
  """
1360
- Specifies that the step will success under all circumstances.
1353
+ Specifies the resources needed when executing this step.
1361
1354
 
1362
- The decorator will create an optional artifact, specified by `var`, which
1363
- contains the exception raised. You can use it to detect the presence
1364
- of errors, indicating that all happy-path artifacts produced by the step
1365
- are missing.
1355
+ Use `@resources` to specify the resource requirements
1356
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1357
+
1358
+ You can choose the compute layer on the command line by executing e.g.
1359
+ ```
1360
+ python myflow.py run --with batch
1361
+ ```
1362
+ or
1363
+ ```
1364
+ python myflow.py run --with kubernetes
1365
+ ```
1366
+ which executes the flow on the desired system using the
1367
+ requirements specified in `@resources`.
1366
1368
 
1367
1369
 
1368
1370
  Parameters
1369
1371
  ----------
1370
- var : str, optional, default None
1371
- Name of the artifact in which to store the caught exception.
1372
- If not specified, the exception is not stored.
1373
- print_exception : bool, default True
1374
- Determines whether or not the exception is printed to
1375
- stdout when caught.
1376
- """
1377
- ...
1378
-
1379
- @typing.overload
1380
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1381
- """
1382
- Decorator prototype for all step decorators. This function gets specialized
1383
- and imported for all decorators types by _import_plugin_decorators().
1372
+ cpu : int, default 1
1373
+ Number of CPUs required for this step.
1374
+ gpu : int, optional, default None
1375
+ Number of GPUs required for this step.
1376
+ disk : int, optional, default None
1377
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1378
+ memory : int, default 4096
1379
+ Memory size (in MB) required for this step.
1380
+ shared_memory : int, optional, default None
1381
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1382
+ This parameter maps to the `--shm-size` option in Docker.
1384
1383
  """
1385
1384
  ...
1386
1385
 
1387
- @typing.overload
1388
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1389
- ...
1390
-
1391
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1386
+ def nebius_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1392
1387
  """
1393
- Decorator prototype for all step decorators. This function gets specialized
1394
- and imported for all decorators types by _import_plugin_decorators().
1388
+ `@nebius_s3_proxy` is a Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1389
+ It exists to make it easier for users to know that this decorator should only be used with
1390
+ a Neo Cloud like Nebius. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
1391
+
1392
+
1393
+ Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
1394
+ for S3 read and write requests.
1395
+
1396
+ This decorator requires an integration in the Outerbounds platform that
1397
+ points to an external bucket. It affects S3 operations performed via
1398
+ Metaflow's `get_aws_client` and `S3` within a `@step`.
1399
+
1400
+ Read operations
1401
+ ---------------
1402
+ All read operations pass through the proxy. If an object does not already
1403
+ exist in the external bucket, it is cached there. For example, if code reads
1404
+ from buckets `FOO` and `BAR` using the `S3` interface, objects from both
1405
+ buckets are cached in the external bucket.
1406
+
1407
+ During task execution, all S3‑related read requests are routed through the
1408
+ proxy:
1409
+ - If the object is present in the external object store, the proxy
1410
+ streams it directly from there without accessing the requested origin
1411
+ bucket.
1412
+ - If the object is not present in the external storage, the proxy
1413
+ fetches it from the requested bucket, caches it in the external
1414
+ storage, and streams the response from the origin bucket.
1415
+
1416
+ Warning
1417
+ -------
1418
+ All READ operations (e.g., GetObject, HeadObject) pass through the external
1419
+ bucket regardless of the bucket specified in user code. Even
1420
+ `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
1421
+ external bucket cache.
1422
+
1423
+ Write operations
1424
+ ----------------
1425
+ Write behavior is controlled by the `write_mode` parameter, which determines
1426
+ whether writes also persist objects in the cache.
1427
+
1428
+ `write_mode` values:
1429
+ - `origin-and-cache`: objects are written both to the cache and to their
1430
+ intended origin bucket.
1431
+ - `origin`: objects are written only to their intended origin bucket.
1432
+
1433
+
1434
+ Parameters
1435
+ ----------
1436
+ integration_name : str, optional
1437
+ [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
1438
+ that holds the configuration for the external, S3‑compatible object
1439
+ storage bucket. If not specified, the only available S3 proxy
1440
+ integration in the namespace is used (fails if multiple exist).
1441
+ write_mode : str, optional
1442
+ Controls whether writes also go to the external bucket.
1443
+ - `origin` (default)
1444
+ - `origin-and-cache`
1445
+ debug : bool, optional
1446
+ Enables debug logging for proxy operations.
1395
1447
  """
1396
1448
  ...
1397
1449
 
1398
- def nebius_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1450
+ def coreweave_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1399
1451
  """
1400
- `@nebius_s3_proxy` is a Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1452
+ `@coreweave_s3_proxy` is a CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1401
1453
  It exists to make it easier for users to know that this decorator should only be used with
1402
- a Neo Cloud like Nebius. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
1454
+ a Neo Cloud like CoreWeave. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
1403
1455
 
1404
1456
 
1405
1457
  Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
@@ -1460,104 +1512,88 @@ def nebius_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode
1460
1512
  ...
1461
1513
 
1462
1514
  @typing.overload
1463
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1515
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1464
1516
  """
1465
- Specifies the PyPI packages for the step.
1517
+ Specifies the Conda environment for all steps of the flow.
1466
1518
 
1467
- Information in this decorator will augment any
1468
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1469
- you can use `@pypi_base` to set packages required by all
1470
- steps and use `@pypi` to specify step-specific overrides.
1519
+ Use `@conda_base` to set common libraries required by all
1520
+ steps and use `@conda` to specify step-specific additions.
1471
1521
 
1472
1522
 
1473
1523
  Parameters
1474
1524
  ----------
1475
- packages : Dict[str, str], default: {}
1476
- Packages to use for this step. The key is the name of the package
1525
+ packages : Dict[str, str], default {}
1526
+ Packages to use for this flow. The key is the name of the package
1477
1527
  and the value is the version to use.
1478
- python : str, optional, default: None
1528
+ libraries : Dict[str, str], default {}
1529
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1530
+ python : str, optional, default None
1479
1531
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1480
1532
  that the version used will correspond to the version of the Python interpreter used to start the run.
1533
+ disabled : bool, default False
1534
+ If set to True, disables Conda.
1481
1535
  """
1482
1536
  ...
1483
1537
 
1484
1538
  @typing.overload
1485
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1486
- ...
1487
-
1488
- @typing.overload
1489
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1539
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1490
1540
  ...
1491
1541
 
1492
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1542
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1493
1543
  """
1494
- Specifies the PyPI packages for the step.
1544
+ Specifies the Conda environment for all steps of the flow.
1495
1545
 
1496
- Information in this decorator will augment any
1497
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1498
- you can use `@pypi_base` to set packages required by all
1499
- steps and use `@pypi` to specify step-specific overrides.
1546
+ Use `@conda_base` to set common libraries required by all
1547
+ steps and use `@conda` to specify step-specific additions.
1500
1548
 
1501
1549
 
1502
1550
  Parameters
1503
1551
  ----------
1504
- packages : Dict[str, str], default: {}
1505
- Packages to use for this step. The key is the name of the package
1552
+ packages : Dict[str, str], default {}
1553
+ Packages to use for this flow. The key is the name of the package
1506
1554
  and the value is the version to use.
1507
- python : str, optional, default: None
1555
+ libraries : Dict[str, str], default {}
1556
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1557
+ python : str, optional, default None
1508
1558
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1509
1559
  that the version used will correspond to the version of the Python interpreter used to start the run.
1560
+ disabled : bool, default False
1561
+ If set to True, disables Conda.
1510
1562
  """
1511
1563
  ...
1512
1564
 
1513
- @typing.overload
1514
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1565
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1515
1566
  """
1516
- Specifies the times when the flow should be run when running on a
1517
- production scheduler.
1567
+ Specifies what flows belong to the same project.
1568
+
1569
+ A project-specific namespace is created for all flows that
1570
+ use the same `@project(name)`.
1518
1571
 
1519
1572
 
1520
1573
  Parameters
1521
1574
  ----------
1522
- hourly : bool, default False
1523
- Run the workflow hourly.
1524
- daily : bool, default True
1525
- Run the workflow daily.
1526
- weekly : bool, default False
1527
- Run the workflow weekly.
1528
- cron : str, optional, default None
1529
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1530
- specified by this expression.
1531
- timezone : str, optional, default None
1532
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1533
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1534
- """
1535
- ...
1536
-
1537
- @typing.overload
1538
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1539
- ...
1540
-
1541
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1542
- """
1543
- Specifies the times when the flow should be run when running on a
1544
- production scheduler.
1575
+ name : str
1576
+ Project name. Make sure that the name is unique amongst all
1577
+ projects that use the same production scheduler. The name may
1578
+ contain only lowercase alphanumeric characters and underscores.
1545
1579
 
1580
+ branch : Optional[str], default None
1581
+ The branch to use. If not specified, the branch is set to
1582
+ `user.<username>` unless `production` is set to `True`. This can
1583
+ also be set on the command line using `--branch` as a top-level option.
1584
+ It is an error to specify `branch` in the decorator and on the command line.
1546
1585
 
1547
- Parameters
1548
- ----------
1549
- hourly : bool, default False
1550
- Run the workflow hourly.
1551
- daily : bool, default True
1552
- Run the workflow daily.
1553
- weekly : bool, default False
1554
- Run the workflow weekly.
1555
- cron : str, optional, default None
1556
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1557
- specified by this expression.
1558
- timezone : str, optional, default None
1559
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1560
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1586
+ production : bool, default False
1587
+ Whether or not the branch is the production branch. This can also be set on the
1588
+ command line using `--production` as a top-level option. It is an error to specify
1589
+ `production` in the decorator and on the command line.
1590
+ The project branch name will be:
1591
+ - if `branch` is specified:
1592
+ - if `production` is True: `prod.<branch>`
1593
+ - if `production` is False: `test.<branch>`
1594
+ - if `branch` is not specified:
1595
+ - if `production` is True: `prod`
1596
+ - if `production` is False: `user.<username>`
1561
1597
  """
1562
1598
  ...
1563
1599
 
@@ -1605,88 +1641,144 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1605
1641
  ...
1606
1642
 
1607
1643
  @typing.overload
1608
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1644
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1609
1645
  """
1610
- Specifies the Conda environment for all steps of the flow.
1646
+ Specifies the flow(s) that this flow depends on.
1611
1647
 
1612
- Use `@conda_base` to set common libraries required by all
1613
- steps and use `@conda` to specify step-specific additions.
1648
+ ```
1649
+ @trigger_on_finish(flow='FooFlow')
1650
+ ```
1651
+ or
1652
+ ```
1653
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1654
+ ```
1655
+ This decorator respects the @project decorator and triggers the flow
1656
+ when upstream runs within the same namespace complete successfully
1657
+
1658
+ Additionally, you can specify project aware upstream flow dependencies
1659
+ by specifying the fully qualified project_flow_name.
1660
+ ```
1661
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1662
+ ```
1663
+ or
1664
+ ```
1665
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1666
+ ```
1667
+
1668
+ You can also specify just the project or project branch (other values will be
1669
+ inferred from the current project or project branch):
1670
+ ```
1671
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1672
+ ```
1673
+
1674
+ Note that `branch` is typically one of:
1675
+ - `prod`
1676
+ - `user.bob`
1677
+ - `test.my_experiment`
1678
+ - `prod.staging`
1679
+
1680
+
1681
+ Parameters
1682
+ ----------
1683
+ flow : Union[str, Dict[str, str]], optional, default None
1684
+ Upstream flow dependency for this flow.
1685
+ flows : List[Union[str, Dict[str, str]]], default []
1686
+ Upstream flow dependencies for this flow.
1687
+ options : Dict[str, Any], default {}
1688
+ Backend-specific configuration for tuning eventing behavior.
1689
+ """
1690
+ ...
1691
+
1692
+ @typing.overload
1693
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1694
+ ...
1695
+
1696
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1697
+ """
1698
+ Specifies the flow(s) that this flow depends on.
1699
+
1700
+ ```
1701
+ @trigger_on_finish(flow='FooFlow')
1702
+ ```
1703
+ or
1704
+ ```
1705
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1706
+ ```
1707
+ This decorator respects the @project decorator and triggers the flow
1708
+ when upstream runs within the same namespace complete successfully
1709
+
1710
+ Additionally, you can specify project aware upstream flow dependencies
1711
+ by specifying the fully qualified project_flow_name.
1712
+ ```
1713
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1714
+ ```
1715
+ or
1716
+ ```
1717
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1718
+ ```
1719
+
1720
+ You can also specify just the project or project branch (other values will be
1721
+ inferred from the current project or project branch):
1722
+ ```
1723
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1724
+ ```
1725
+
1726
+ Note that `branch` is typically one of:
1727
+ - `prod`
1728
+ - `user.bob`
1729
+ - `test.my_experiment`
1730
+ - `prod.staging`
1731
+
1732
+
1733
+ Parameters
1734
+ ----------
1735
+ flow : Union[str, Dict[str, str]], optional, default None
1736
+ Upstream flow dependency for this flow.
1737
+ flows : List[Union[str, Dict[str, str]]], default []
1738
+ Upstream flow dependencies for this flow.
1739
+ options : Dict[str, Any], default {}
1740
+ Backend-specific configuration for tuning eventing behavior.
1741
+ """
1742
+ ...
1743
+
1744
+ @typing.overload
1745
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1746
+ """
1747
+ Specifies the PyPI packages for all steps of the flow.
1614
1748
 
1749
+ Use `@pypi_base` to set common packages required by all
1750
+ steps and use `@pypi` to specify step-specific overrides.
1615
1751
 
1616
1752
  Parameters
1617
1753
  ----------
1618
- packages : Dict[str, str], default {}
1754
+ packages : Dict[str, str], default: {}
1619
1755
  Packages to use for this flow. The key is the name of the package
1620
1756
  and the value is the version to use.
1621
- libraries : Dict[str, str], default {}
1622
- Supported for backward compatibility. When used with packages, packages will take precedence.
1623
- python : str, optional, default None
1757
+ python : str, optional, default: None
1624
1758
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1625
1759
  that the version used will correspond to the version of the Python interpreter used to start the run.
1626
- disabled : bool, default False
1627
- If set to True, disables Conda.
1628
1760
  """
1629
1761
  ...
1630
1762
 
1631
1763
  @typing.overload
1632
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1764
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1633
1765
  ...
1634
1766
 
1635
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1767
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1636
1768
  """
1637
- Specifies the Conda environment for all steps of the flow.
1638
-
1639
- Use `@conda_base` to set common libraries required by all
1640
- steps and use `@conda` to specify step-specific additions.
1769
+ Specifies the PyPI packages for all steps of the flow.
1641
1770
 
1771
+ Use `@pypi_base` to set common packages required by all
1772
+ steps and use `@pypi` to specify step-specific overrides.
1642
1773
 
1643
1774
  Parameters
1644
1775
  ----------
1645
- packages : Dict[str, str], default {}
1776
+ packages : Dict[str, str], default: {}
1646
1777
  Packages to use for this flow. The key is the name of the package
1647
1778
  and the value is the version to use.
1648
- libraries : Dict[str, str], default {}
1649
- Supported for backward compatibility. When used with packages, packages will take precedence.
1650
- python : str, optional, default None
1779
+ python : str, optional, default: None
1651
1780
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1652
1781
  that the version used will correspond to the version of the Python interpreter used to start the run.
1653
- disabled : bool, default False
1654
- If set to True, disables Conda.
1655
- """
1656
- ...
1657
-
1658
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1659
- """
1660
- Specifies what flows belong to the same project.
1661
-
1662
- A project-specific namespace is created for all flows that
1663
- use the same `@project(name)`.
1664
-
1665
-
1666
- Parameters
1667
- ----------
1668
- name : str
1669
- Project name. Make sure that the name is unique amongst all
1670
- projects that use the same production scheduler. The name may
1671
- contain only lowercase alphanumeric characters and underscores.
1672
-
1673
- branch : Optional[str], default None
1674
- The branch to use. If not specified, the branch is set to
1675
- `user.<username>` unless `production` is set to `True`. This can
1676
- also be set on the command line using `--branch` as a top-level option.
1677
- It is an error to specify `branch` in the decorator and on the command line.
1678
-
1679
- production : bool, default False
1680
- Whether or not the branch is the production branch. This can also be set on the
1681
- command line using `--production` as a top-level option. It is an error to specify
1682
- `production` in the decorator and on the command line.
1683
- The project branch name will be:
1684
- - if `branch` is specified:
1685
- - if `production` is True: `prod.<branch>`
1686
- - if `production` is False: `test.<branch>`
1687
- - if `branch` is not specified:
1688
- - if `production` is True: `prod`
1689
- - if `production` is False: `user.<username>`
1690
1782
  """
1691
1783
  ...
1692
1784
 
@@ -1847,148 +1939,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1847
1939
  """
1848
1940
  ...
1849
1941
 
1850
- @typing.overload
1851
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1852
- """
1853
- Specifies the flow(s) that this flow depends on.
1854
-
1855
- ```
1856
- @trigger_on_finish(flow='FooFlow')
1857
- ```
1858
- or
1859
- ```
1860
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1861
- ```
1862
- This decorator respects the @project decorator and triggers the flow
1863
- when upstream runs within the same namespace complete successfully
1864
-
1865
- Additionally, you can specify project aware upstream flow dependencies
1866
- by specifying the fully qualified project_flow_name.
1867
- ```
1868
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1869
- ```
1870
- or
1871
- ```
1872
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1873
- ```
1874
-
1875
- You can also specify just the project or project branch (other values will be
1876
- inferred from the current project or project branch):
1877
- ```
1878
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1879
- ```
1880
-
1881
- Note that `branch` is typically one of:
1882
- - `prod`
1883
- - `user.bob`
1884
- - `test.my_experiment`
1885
- - `prod.staging`
1886
-
1887
-
1888
- Parameters
1889
- ----------
1890
- flow : Union[str, Dict[str, str]], optional, default None
1891
- Upstream flow dependency for this flow.
1892
- flows : List[Union[str, Dict[str, str]]], default []
1893
- Upstream flow dependencies for this flow.
1894
- options : Dict[str, Any], default {}
1895
- Backend-specific configuration for tuning eventing behavior.
1896
- """
1897
- ...
1898
-
1899
- @typing.overload
1900
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1901
- ...
1902
-
1903
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1904
- """
1905
- Specifies the flow(s) that this flow depends on.
1906
-
1907
- ```
1908
- @trigger_on_finish(flow='FooFlow')
1909
- ```
1910
- or
1911
- ```
1912
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1913
- ```
1914
- This decorator respects the @project decorator and triggers the flow
1915
- when upstream runs within the same namespace complete successfully
1916
-
1917
- Additionally, you can specify project aware upstream flow dependencies
1918
- by specifying the fully qualified project_flow_name.
1919
- ```
1920
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1921
- ```
1922
- or
1923
- ```
1924
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1925
- ```
1926
-
1927
- You can also specify just the project or project branch (other values will be
1928
- inferred from the current project or project branch):
1929
- ```
1930
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1931
- ```
1932
-
1933
- Note that `branch` is typically one of:
1934
- - `prod`
1935
- - `user.bob`
1936
- - `test.my_experiment`
1937
- - `prod.staging`
1938
-
1939
-
1940
- Parameters
1941
- ----------
1942
- flow : Union[str, Dict[str, str]], optional, default None
1943
- Upstream flow dependency for this flow.
1944
- flows : List[Union[str, Dict[str, str]]], default []
1945
- Upstream flow dependencies for this flow.
1946
- options : Dict[str, Any], default {}
1947
- Backend-specific configuration for tuning eventing behavior.
1948
- """
1949
- ...
1950
-
1951
- @typing.overload
1952
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1953
- """
1954
- Specifies the PyPI packages for all steps of the flow.
1955
-
1956
- Use `@pypi_base` to set common packages required by all
1957
- steps and use `@pypi` to specify step-specific overrides.
1958
-
1959
- Parameters
1960
- ----------
1961
- packages : Dict[str, str], default: {}
1962
- Packages to use for this flow. The key is the name of the package
1963
- and the value is the version to use.
1964
- python : str, optional, default: None
1965
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1966
- that the version used will correspond to the version of the Python interpreter used to start the run.
1967
- """
1968
- ...
1969
-
1970
- @typing.overload
1971
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1972
- ...
1973
-
1974
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1975
- """
1976
- Specifies the PyPI packages for all steps of the flow.
1977
-
1978
- Use `@pypi_base` to set common packages required by all
1979
- steps and use `@pypi` to specify step-specific overrides.
1980
-
1981
- Parameters
1982
- ----------
1983
- packages : Dict[str, str], default: {}
1984
- Packages to use for this flow. The key is the name of the package
1985
- and the value is the version to use.
1986
- python : str, optional, default: None
1987
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1988
- that the version used will correspond to the version of the Python interpreter used to start the run.
1989
- """
1990
- ...
1991
-
1992
1942
  @typing.overload
1993
1943
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1994
1944
  """
@@ -2082,5 +2032,56 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
2082
2032
  """
2083
2033
  ...
2084
2034
 
2035
+ @typing.overload
2036
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
2037
+ """
2038
+ Specifies the times when the flow should be run when running on a
2039
+ production scheduler.
2040
+
2041
+
2042
+ Parameters
2043
+ ----------
2044
+ hourly : bool, default False
2045
+ Run the workflow hourly.
2046
+ daily : bool, default True
2047
+ Run the workflow daily.
2048
+ weekly : bool, default False
2049
+ Run the workflow weekly.
2050
+ cron : str, optional, default None
2051
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
2052
+ specified by this expression.
2053
+ timezone : str, optional, default None
2054
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
2055
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
2056
+ """
2057
+ ...
2058
+
2059
+ @typing.overload
2060
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
2061
+ ...
2062
+
2063
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
2064
+ """
2065
+ Specifies the times when the flow should be run when running on a
2066
+ production scheduler.
2067
+
2068
+
2069
+ Parameters
2070
+ ----------
2071
+ hourly : bool, default False
2072
+ Run the workflow hourly.
2073
+ daily : bool, default True
2074
+ Run the workflow daily.
2075
+ weekly : bool, default False
2076
+ Run the workflow weekly.
2077
+ cron : str, optional, default None
2078
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
2079
+ specified by this expression.
2080
+ timezone : str, optional, default None
2081
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
2082
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
2083
+ """
2084
+ ...
2085
+
2085
2086
  pkg_name: str
2086
2087