ob-metaflow-stubs 6.0.5.1__py2.py3-none-any.whl → 6.0.6.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (261) hide show
  1. metaflow-stubs/__init__.pyi +927 -927
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +2 -2
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +32 -32
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +3 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +3 -3
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +2 -2
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +7 -3
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +11 -2
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +8 -9
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +3 -3
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +3 -3
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +4 -4
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  110. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  111. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  112. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  113. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  114. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
  116. metaflow-stubs/multicore_utils.pyi +2 -2
  117. metaflow-stubs/ob_internal.pyi +2 -2
  118. metaflow-stubs/packaging_sys/__init__.pyi +6 -6
  119. metaflow-stubs/packaging_sys/backend.pyi +3 -3
  120. metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
  121. metaflow-stubs/packaging_sys/tar_backend.pyi +6 -6
  122. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  123. metaflow-stubs/packaging_sys/v1.pyi +2 -2
  124. metaflow-stubs/parameters.pyi +2 -2
  125. metaflow-stubs/plugins/__init__.pyi +14 -14
  126. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  128. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  129. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  130. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  131. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  132. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  133. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  134. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  135. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  136. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  137. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  138. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  139. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  140. metaflow-stubs/plugins/argo/exit_hooks.pyi +2 -2
  141. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  142. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  143. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  144. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  146. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  147. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  148. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +5 -5
  150. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  152. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  154. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  155. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  157. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  158. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  159. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  160. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +5 -5
  161. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  162. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  163. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  164. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  165. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  166. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  167. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  168. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  169. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  170. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  171. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  172. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  173. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  174. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  175. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  176. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  177. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  178. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  179. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  180. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  181. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  182. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  183. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  184. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  185. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  186. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  187. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  188. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  189. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  190. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  191. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  192. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  193. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  194. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  195. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +5 -5
  196. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  197. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  198. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  199. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  200. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  201. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  202. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  203. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  204. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  205. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  206. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  207. metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
  208. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  209. metaflow-stubs/plugins/perimeters.pyi +2 -2
  210. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  211. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  212. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  213. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  214. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  215. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  216. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  217. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  218. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  219. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  220. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  221. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  222. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  223. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  224. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  225. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  226. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  227. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  228. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  229. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  230. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  231. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  232. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  233. metaflow-stubs/profilers/__init__.pyi +2 -2
  234. metaflow-stubs/pylint_wrapper.pyi +2 -2
  235. metaflow-stubs/runner/__init__.pyi +2 -2
  236. metaflow-stubs/runner/deployer.pyi +32 -32
  237. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  238. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  239. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  240. metaflow-stubs/runner/nbrun.pyi +2 -2
  241. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  242. metaflow-stubs/runner/utils.pyi +3 -3
  243. metaflow-stubs/system/__init__.pyi +2 -2
  244. metaflow-stubs/system/system_logger.pyi +3 -3
  245. metaflow-stubs/system/system_monitor.pyi +2 -2
  246. metaflow-stubs/tagging_util.pyi +2 -2
  247. metaflow-stubs/tuple_util.pyi +2 -2
  248. metaflow-stubs/user_configs/__init__.pyi +2 -2
  249. metaflow-stubs/user_configs/config_options.pyi +2 -2
  250. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  251. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  252. metaflow-stubs/user_decorators/common.pyi +2 -2
  253. metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
  254. metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
  255. metaflow-stubs/user_decorators/user_flow_decorator.pyi +3 -3
  256. metaflow-stubs/user_decorators/user_step_decorator.pyi +5 -5
  257. {ob_metaflow_stubs-6.0.5.1.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/METADATA +1 -1
  258. ob_metaflow_stubs-6.0.6.0.dist-info/RECORD +261 -0
  259. ob_metaflow_stubs-6.0.5.1.dist-info/RECORD +0 -261
  260. {ob_metaflow_stubs-6.0.5.1.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/WHEEL +0 -0
  261. {ob_metaflow_stubs-6.0.5.1.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.16.8.1+obcheckpoint(0.2.4);ob(v1) #
4
- # Generated on 2025-08-01T20:12:28.874985 #
3
+ # MF version: 2.17.0.1+obcheckpoint(0.2.4);ob(v1) #
4
+ # Generated on 2025-08-05T23:30:10.180789 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -39,8 +39,8 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
39
39
  from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
- from . import cards as cards
43
42
  from . import metaflow_git as metaflow_git
43
+ from . import cards as cards
44
44
  from . import tuple_util as tuple_util
45
45
  from . import events as events
46
46
  from . import runner as runner
@@ -48,9 +48,9 @@ from . import plugins as plugins
48
48
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
49
49
  from . import includefile as includefile
50
50
  from .includefile import IncludeFile as IncludeFile
51
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
52
51
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
53
52
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
53
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
54
54
  from . import client as client
55
55
  from .client.core import namespace as namespace
56
56
  from .client.core import get_namespace as get_namespace
@@ -168,92 +168,35 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
168
168
  ...
169
169
 
170
170
  @typing.overload
171
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
172
- """
173
- Specifies that the step will success under all circumstances.
174
-
175
- The decorator will create an optional artifact, specified by `var`, which
176
- contains the exception raised. You can use it to detect the presence
177
- of errors, indicating that all happy-path artifacts produced by the step
178
- are missing.
179
-
180
-
181
- Parameters
182
- ----------
183
- var : str, optional, default None
184
- Name of the artifact in which to store the caught exception.
185
- If not specified, the exception is not stored.
186
- print_exception : bool, default True
187
- Determines whether or not the exception is printed to
188
- stdout when caught.
189
- """
190
- ...
191
-
192
- @typing.overload
193
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
194
- ...
195
-
196
- @typing.overload
197
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
198
- ...
199
-
200
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
201
- """
202
- Specifies that the step will success under all circumstances.
203
-
204
- The decorator will create an optional artifact, specified by `var`, which
205
- contains the exception raised. You can use it to detect the presence
206
- of errors, indicating that all happy-path artifacts produced by the step
207
- are missing.
208
-
209
-
210
- Parameters
211
- ----------
212
- var : str, optional, default None
213
- Name of the artifact in which to store the caught exception.
214
- If not specified, the exception is not stored.
215
- print_exception : bool, default True
216
- Determines whether or not the exception is printed to
217
- stdout when caught.
218
- """
219
- ...
220
-
221
- @typing.overload
222
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
171
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
223
172
  """
224
- Specifies secrets to be retrieved and injected as environment variables prior to
225
- the execution of a step.
173
+ Specifies environment variables to be set prior to the execution of a step.
226
174
 
227
175
 
228
176
  Parameters
229
177
  ----------
230
- sources : List[Union[str, Dict[str, Any]]], default: []
231
- List of secret specs, defining how the secrets are to be retrieved
232
- role : str, optional, default: None
233
- Role to use for fetching secrets
178
+ vars : Dict[str, str], default {}
179
+ Dictionary of environment variables to set.
234
180
  """
235
181
  ...
236
182
 
237
183
  @typing.overload
238
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
184
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
239
185
  ...
240
186
 
241
187
  @typing.overload
242
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
188
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
243
189
  ...
244
190
 
245
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
191
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
246
192
  """
247
- Specifies secrets to be retrieved and injected as environment variables prior to
248
- the execution of a step.
193
+ Specifies environment variables to be set prior to the execution of a step.
249
194
 
250
195
 
251
196
  Parameters
252
197
  ----------
253
- sources : List[Union[str, Dict[str, Any]]], default: []
254
- List of secret specs, defining how the secrets are to be retrieved
255
- role : str, optional, default: None
256
- Role to use for fetching secrets
198
+ vars : Dict[str, str], default {}
199
+ Dictionary of environment variables to set.
257
200
  """
258
201
  ...
259
202
 
@@ -275,21 +218,57 @@ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepF
275
218
  ...
276
219
 
277
220
  @typing.overload
278
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
221
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
279
222
  """
280
- A simple decorator that demonstrates using CardDecoratorInjector
281
- to inject a card and render simple markdown content.
223
+ Specifies the number of times the task corresponding
224
+ to a step needs to be retried.
225
+
226
+ This decorator is useful for handling transient errors, such as networking issues.
227
+ If your task contains operations that can't be retried safely, e.g. database updates,
228
+ it is advisable to annotate it with `@retry(times=0)`.
229
+
230
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
231
+ decorator will execute a no-op task after all retries have been exhausted,
232
+ ensuring that the flow execution can continue.
233
+
234
+
235
+ Parameters
236
+ ----------
237
+ times : int, default 3
238
+ Number of times to retry this task.
239
+ minutes_between_retries : int, default 2
240
+ Number of minutes between retries.
282
241
  """
283
242
  ...
284
243
 
285
244
  @typing.overload
286
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
245
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
287
246
  ...
288
247
 
289
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
248
+ @typing.overload
249
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
250
+ ...
251
+
252
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
290
253
  """
291
- A simple decorator that demonstrates using CardDecoratorInjector
292
- to inject a card and render simple markdown content.
254
+ Specifies the number of times the task corresponding
255
+ to a step needs to be retried.
256
+
257
+ This decorator is useful for handling transient errors, such as networking issues.
258
+ If your task contains operations that can't be retried safely, e.g. database updates,
259
+ it is advisable to annotate it with `@retry(times=0)`.
260
+
261
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
262
+ decorator will execute a no-op task after all retries have been exhausted,
263
+ ensuring that the flow execution can continue.
264
+
265
+
266
+ Parameters
267
+ ----------
268
+ times : int, default 3
269
+ Number of times to retry this task.
270
+ minutes_between_retries : int, default 2
271
+ Number of minutes between retries.
293
272
  """
294
273
  ...
295
274
 
@@ -344,278 +323,83 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
344
323
  """
345
324
  ...
346
325
 
347
- def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
326
+ @typing.overload
327
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
348
328
  """
349
- This decorator is used to run vllm APIs as Metaflow task sidecars.
350
-
351
- User code call
352
- --------------
353
- @vllm(
354
- model="...",
355
- ...
356
- )
329
+ Specifies a timeout for your step.
357
330
 
358
- Valid backend options
359
- ---------------------
360
- - 'local': Run as a separate process on the local task machine.
331
+ This decorator is useful if this step may hang indefinitely.
361
332
 
362
- Valid model options
363
- -------------------
364
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
333
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
334
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
335
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
365
336
 
366
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
367
- If you need multiple models, you must create multiple @vllm decorators.
337
+ Note that all the values specified in parameters are added together so if you specify
338
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
368
339
 
369
340
 
370
341
  Parameters
371
342
  ----------
372
- model: str
373
- HuggingFace model identifier to be served by vLLM.
374
- backend: str
375
- Determines where and how to run the vLLM process.
376
- openai_api_server: bool
377
- Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
378
- Default is False (uses native engine).
379
- Set to True for backward compatibility with existing code.
380
- debug: bool
381
- Whether to turn on verbose debugging logs.
382
- card_refresh_interval: int
383
- Interval in seconds for refreshing the vLLM status card.
384
- Only used when openai_api_server=True.
385
- max_retries: int
386
- Maximum number of retries checking for vLLM server startup.
387
- Only used when openai_api_server=True.
388
- retry_alert_frequency: int
389
- Frequency of alert logs for vLLM server startup retries.
390
- Only used when openai_api_server=True.
391
- engine_args : dict
392
- Additional keyword arguments to pass to the vLLM engine.
393
- For example, `tensor_parallel_size=2`.
343
+ seconds : int, default 0
344
+ Number of seconds to wait prior to timing out.
345
+ minutes : int, default 0
346
+ Number of minutes to wait prior to timing out.
347
+ hours : int, default 0
348
+ Number of hours to wait prior to timing out.
394
349
  """
395
350
  ...
396
351
 
397
352
  @typing.overload
398
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
353
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
354
+ ...
355
+
356
+ @typing.overload
357
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
358
+ ...
359
+
360
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
399
361
  """
400
- Enables loading / saving of models within a step.
362
+ Specifies a timeout for your step.
401
363
 
402
- > Examples
403
- - Saving Models
404
- ```python
405
- @model
406
- @step
407
- def train(self):
408
- # current.model.save returns a dictionary reference to the model saved
409
- self.my_model = current.model.save(
410
- path_to_my_model,
411
- label="my_model",
412
- metadata={
413
- "epochs": 10,
414
- "batch-size": 32,
415
- "learning-rate": 0.001,
416
- }
417
- )
418
- self.next(self.test)
364
+ This decorator is useful if this step may hang indefinitely.
419
365
 
420
- @model(load="my_model")
421
- @step
422
- def test(self):
423
- # `current.model.loaded` returns a dictionary of the loaded models
424
- # where the key is the name of the artifact and the value is the path to the model
425
- print(os.listdir(current.model.loaded["my_model"]))
426
- self.next(self.end)
427
- ```
366
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
367
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
368
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
428
369
 
429
- - Loading models
430
- ```python
431
- @step
432
- def train(self):
433
- # current.model.load returns the path to the model loaded
434
- checkpoint_path = current.model.load(
435
- self.checkpoint_key,
436
- )
437
- model_path = current.model.load(
438
- self.model,
439
- )
440
- self.next(self.test)
441
- ```
370
+ Note that all the values specified in parameters are added together so if you specify
371
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
442
372
 
443
373
 
444
374
  Parameters
445
375
  ----------
446
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
447
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
448
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
449
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
450
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
451
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
452
-
453
- temp_dir_root : str, default: None
454
- The root directory under which `current.model.loaded` will store loaded models
376
+ seconds : int, default 0
377
+ Number of seconds to wait prior to timing out.
378
+ minutes : int, default 0
379
+ Number of minutes to wait prior to timing out.
380
+ hours : int, default 0
381
+ Number of hours to wait prior to timing out.
455
382
  """
456
383
  ...
457
384
 
458
385
  @typing.overload
459
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
460
- ...
461
-
462
- @typing.overload
463
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
464
- ...
465
-
466
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
386
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
467
387
  """
468
- Enables loading / saving of models within a step.
469
-
470
- > Examples
471
- - Saving Models
472
- ```python
473
- @model
474
- @step
475
- def train(self):
476
- # current.model.save returns a dictionary reference to the model saved
477
- self.my_model = current.model.save(
478
- path_to_my_model,
479
- label="my_model",
480
- metadata={
481
- "epochs": 10,
482
- "batch-size": 32,
483
- "learning-rate": 0.001,
484
- }
485
- )
486
- self.next(self.test)
487
-
488
- @model(load="my_model")
489
- @step
490
- def test(self):
491
- # `current.model.loaded` returns a dictionary of the loaded models
492
- # where the key is the name of the artifact and the value is the path to the model
493
- print(os.listdir(current.model.loaded["my_model"]))
494
- self.next(self.end)
495
- ```
496
-
497
- - Loading models
498
- ```python
499
- @step
500
- def train(self):
501
- # current.model.load returns the path to the model loaded
502
- checkpoint_path = current.model.load(
503
- self.checkpoint_key,
504
- )
505
- model_path = current.model.load(
506
- self.model,
507
- )
508
- self.next(self.test)
509
- ```
510
-
511
-
512
- Parameters
513
- ----------
514
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
515
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
516
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
517
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
518
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
519
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
520
-
521
- temp_dir_root : str, default: None
522
- The root directory under which `current.model.loaded` will store loaded models
388
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
389
+ It exists to make it easier for users to know that this decorator should only be used with
390
+ a Neo Cloud like Nebius.
523
391
  """
524
392
  ...
525
393
 
526
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
527
- """
528
- Decorator that helps cache, version and store models/datasets from huggingface hub.
529
-
530
- > Examples
531
-
532
- **Usage: creating references of models from huggingface that may be loaded in downstream steps**
533
- ```python
534
- @huggingface_hub
535
- @step
536
- def pull_model_from_huggingface(self):
537
- # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
538
- # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
539
- # with the same `repo_id` in the backend storage, it will not download the model again. The return
540
- # value of the function is a reference to the model in the backend storage.
541
- # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
542
-
543
- self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
544
- self.llama_model = current.huggingface_hub.snapshot_download(
545
- repo_id=self.model_id,
546
- allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
547
- )
548
- self.next(self.train)
549
- ```
550
-
551
- **Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
552
- ```python
553
- @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
554
- @step
555
- def pull_model_from_huggingface(self):
556
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
557
- ```
558
-
559
- ```python
560
- @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
561
- @step
562
- def finetune_model(self):
563
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
564
- # path_to_model will be /my-directory
565
- ```
566
-
567
- ```python
568
- # Takes all the arguments passed to `snapshot_download`
569
- # except for `local_dir`
570
- @huggingface_hub(load=[
571
- {
572
- "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
573
- },
574
- {
575
- "repo_id": "myorg/mistral-lora",
576
- "repo_type": "model",
577
- },
578
- ])
579
- @step
580
- def finetune_model(self):
581
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
582
- # path_to_model will be /my-directory
583
- ```
584
-
585
-
586
- Parameters
587
- ----------
588
- temp_dir_root : str, optional
589
- The root directory that will hold the temporary directory where objects will be downloaded.
590
-
591
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
592
- The list of repos (models/datasets) to load.
593
-
594
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
595
-
596
- - If repo (model/dataset) is not found in the datastore:
597
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
598
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
599
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
600
-
601
- - If repo is found in the datastore:
602
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
603
- """
394
+ @typing.overload
395
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
604
396
  ...
605
397
 
606
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
398
+ def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
607
399
  """
608
- Specifies that this step should execute on DGX cloud.
609
-
610
-
611
- Parameters
612
- ----------
613
- gpu : int
614
- Number of GPUs to use.
615
- gpu_type : str
616
- Type of Nvidia GPU to use.
617
- queue_timeout : int
618
- Time to keep the job in NVCF's queue.
400
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
401
+ It exists to make it easier for users to know that this decorator should only be used with
402
+ a Neo Cloud like Nebius.
619
403
  """
620
404
  ...
621
405
 
@@ -766,58 +550,6 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
766
550
  """
767
551
  ...
768
552
 
769
- @typing.overload
770
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
771
- """
772
- Decorator prototype for all step decorators. This function gets specialized
773
- and imported for all decorators types by _import_plugin_decorators().
774
- """
775
- ...
776
-
777
- @typing.overload
778
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
779
- ...
780
-
781
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
782
- """
783
- Decorator prototype for all step decorators. This function gets specialized
784
- and imported for all decorators types by _import_plugin_decorators().
785
- """
786
- ...
787
-
788
- @typing.overload
789
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
790
- """
791
- Specifies environment variables to be set prior to the execution of a step.
792
-
793
-
794
- Parameters
795
- ----------
796
- vars : Dict[str, str], default {}
797
- Dictionary of environment variables to set.
798
- """
799
- ...
800
-
801
- @typing.overload
802
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
803
- ...
804
-
805
- @typing.overload
806
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
807
- ...
808
-
809
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
810
- """
811
- Specifies environment variables to be set prior to the execution of a step.
812
-
813
-
814
- Parameters
815
- ----------
816
- vars : Dict[str, str], default {}
817
- Dictionary of environment variables to set.
818
- """
819
- ...
820
-
821
553
  def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
822
554
  """
823
555
  Specifies that this step should execute on Kubernetes.
@@ -907,93 +639,102 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
907
639
  """
908
640
  ...
909
641
 
910
- @typing.overload
911
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
912
- """
913
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
914
- It exists to make it easier for users to know that this decorator should only be used with
915
- a Neo Cloud like CoreWeave.
916
- """
917
- ...
918
-
919
- @typing.overload
920
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
921
- ...
922
-
923
- def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
924
- """
925
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
926
- It exists to make it easier for users to know that this decorator should only be used with
927
- a Neo Cloud like CoreWeave.
928
- """
929
- ...
930
-
931
- @typing.overload
932
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
642
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
933
643
  """
934
- Specifies the number of times the task corresponding
935
- to a step needs to be retried.
936
-
937
- This decorator is useful for handling transient errors, such as networking issues.
938
- If your task contains operations that can't be retried safely, e.g. database updates,
939
- it is advisable to annotate it with `@retry(times=0)`.
940
-
941
- This can be used in conjunction with the `@catch` decorator. The `@catch`
942
- decorator will execute a no-op task after all retries have been exhausted,
943
- ensuring that the flow execution can continue.
644
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
944
645
 
646
+ > Examples
945
647
 
946
- Parameters
947
- ----------
948
- times : int, default 3
949
- Number of times to retry this task.
950
- minutes_between_retries : int, default 2
951
- Number of minutes between retries.
952
- """
953
- ...
954
-
955
- @typing.overload
956
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
957
- ...
958
-
959
- @typing.overload
960
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
961
- ...
962
-
963
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
964
- """
965
- Specifies the number of times the task corresponding
966
- to a step needs to be retried.
648
+ **Usage: creating references of models from huggingface that may be loaded in downstream steps**
649
+ ```python
650
+ @huggingface_hub
651
+ @step
652
+ def pull_model_from_huggingface(self):
653
+ # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
654
+ # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
655
+ # with the same `repo_id` in the backend storage, it will not download the model again. The return
656
+ # value of the function is a reference to the model in the backend storage.
657
+ # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
967
658
 
968
- This decorator is useful for handling transient errors, such as networking issues.
969
- If your task contains operations that can't be retried safely, e.g. database updates,
970
- it is advisable to annotate it with `@retry(times=0)`.
659
+ self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
660
+ self.llama_model = current.huggingface_hub.snapshot_download(
661
+ repo_id=self.model_id,
662
+ allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
663
+ )
664
+ self.next(self.train)
665
+ ```
971
666
 
972
- This can be used in conjunction with the `@catch` decorator. The `@catch`
973
- decorator will execute a no-op task after all retries have been exhausted,
974
- ensuring that the flow execution can continue.
667
+ **Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
668
+ ```python
669
+ @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
670
+ @step
671
+ def pull_model_from_huggingface(self):
672
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
673
+ ```
674
+
675
+ ```python
676
+ @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
677
+ @step
678
+ def finetune_model(self):
679
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
680
+ # path_to_model will be /my-directory
681
+ ```
682
+
683
+ ```python
684
+ # Takes all the arguments passed to `snapshot_download`
685
+ # except for `local_dir`
686
+ @huggingface_hub(load=[
687
+ {
688
+ "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
689
+ },
690
+ {
691
+ "repo_id": "myorg/mistral-lora",
692
+ "repo_type": "model",
693
+ },
694
+ ])
695
+ @step
696
+ def finetune_model(self):
697
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
698
+ # path_to_model will be /my-directory
699
+ ```
975
700
 
976
701
 
977
702
  Parameters
978
703
  ----------
979
- times : int, default 3
980
- Number of times to retry this task.
981
- minutes_between_retries : int, default 2
982
- Number of minutes between retries.
704
+ temp_dir_root : str, optional
705
+ The root directory that will hold the temporary directory where objects will be downloaded.
706
+
707
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
708
+ The list of repos (models/datasets) to load.
709
+
710
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
711
+
712
+ - If repo (model/dataset) is not found in the datastore:
713
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
714
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
715
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
716
+
717
+ - If repo is found in the datastore:
718
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
983
719
  """
984
720
  ...
985
721
 
986
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
722
+ @typing.overload
723
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
987
724
  """
988
- Specifies that this step should execute on DGX cloud.
989
-
990
-
991
- Parameters
992
- ----------
993
- gpu : int
994
- Number of GPUs to use.
995
- gpu_type : str
996
- Type of Nvidia GPU to use.
725
+ Decorator prototype for all step decorators. This function gets specialized
726
+ and imported for all decorators types by _import_plugin_decorators().
727
+ """
728
+ ...
729
+
730
+ @typing.overload
731
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
732
+ ...
733
+
734
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
735
+ """
736
+ Decorator prototype for all step decorators. This function gets specialized
737
+ and imported for all decorators types by _import_plugin_decorators().
997
738
  """
998
739
  ...
999
740
 
@@ -1047,202 +788,223 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
1047
788
  ...
1048
789
 
1049
790
  @typing.overload
1050
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
791
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1051
792
  """
1052
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1053
- It exists to make it easier for users to know that this decorator should only be used with
1054
- a Neo Cloud like Nebius.
793
+ A simple decorator that demonstrates using CardDecoratorInjector
794
+ to inject a card and render simple markdown content.
1055
795
  """
1056
796
  ...
1057
797
 
1058
798
  @typing.overload
1059
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
799
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1060
800
  ...
1061
801
 
1062
- def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
802
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1063
803
  """
1064
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1065
- It exists to make it easier for users to know that this decorator should only be used with
1066
- a Neo Cloud like Nebius.
804
+ A simple decorator that demonstrates using CardDecoratorInjector
805
+ to inject a card and render simple markdown content.
1067
806
  """
1068
807
  ...
1069
808
 
1070
- @typing.overload
1071
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
809
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1072
810
  """
1073
- Specifies the resources needed when executing this step.
1074
-
1075
- Use `@resources` to specify the resource requirements
1076
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1077
-
1078
- You can choose the compute layer on the command line by executing e.g.
1079
- ```
1080
- python myflow.py run --with batch
1081
- ```
1082
- or
1083
- ```
1084
- python myflow.py run --with kubernetes
1085
- ```
1086
- which executes the flow on the desired system using the
1087
- requirements specified in `@resources`.
811
+ Specifies that this step should execute on DGX cloud.
1088
812
 
1089
813
 
1090
814
  Parameters
1091
815
  ----------
1092
- cpu : int, default 1
1093
- Number of CPUs required for this step.
1094
- gpu : int, optional, default None
1095
- Number of GPUs required for this step.
1096
- disk : int, optional, default None
1097
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1098
- memory : int, default 4096
1099
- Memory size (in MB) required for this step.
1100
- shared_memory : int, optional, default None
1101
- The value for the size (in MiB) of the /dev/shm volume for this step.
1102
- This parameter maps to the `--shm-size` option in Docker.
816
+ gpu : int
817
+ Number of GPUs to use.
818
+ gpu_type : str
819
+ Type of Nvidia GPU to use.
1103
820
  """
1104
821
  ...
1105
822
 
1106
823
  @typing.overload
1107
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1108
- ...
1109
-
1110
- @typing.overload
1111
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1112
- ...
1113
-
1114
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
824
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1115
825
  """
1116
- Specifies the resources needed when executing this step.
826
+ Enables loading / saving of models within a step.
1117
827
 
1118
- Use `@resources` to specify the resource requirements
1119
- independently of the specific compute layer (`@batch`, `@kubernetes`).
828
+ > Examples
829
+ - Saving Models
830
+ ```python
831
+ @model
832
+ @step
833
+ def train(self):
834
+ # current.model.save returns a dictionary reference to the model saved
835
+ self.my_model = current.model.save(
836
+ path_to_my_model,
837
+ label="my_model",
838
+ metadata={
839
+ "epochs": 10,
840
+ "batch-size": 32,
841
+ "learning-rate": 0.001,
842
+ }
843
+ )
844
+ self.next(self.test)
1120
845
 
1121
- You can choose the compute layer on the command line by executing e.g.
1122
- ```
1123
- python myflow.py run --with batch
1124
- ```
1125
- or
846
+ @model(load="my_model")
847
+ @step
848
+ def test(self):
849
+ # `current.model.loaded` returns a dictionary of the loaded models
850
+ # where the key is the name of the artifact and the value is the path to the model
851
+ print(os.listdir(current.model.loaded["my_model"]))
852
+ self.next(self.end)
1126
853
  ```
1127
- python myflow.py run --with kubernetes
854
+
855
+ - Loading models
856
+ ```python
857
+ @step
858
+ def train(self):
859
+ # current.model.load returns the path to the model loaded
860
+ checkpoint_path = current.model.load(
861
+ self.checkpoint_key,
862
+ )
863
+ model_path = current.model.load(
864
+ self.model,
865
+ )
866
+ self.next(self.test)
1128
867
  ```
1129
- which executes the flow on the desired system using the
1130
- requirements specified in `@resources`.
1131
868
 
1132
869
 
1133
870
  Parameters
1134
871
  ----------
1135
- cpu : int, default 1
1136
- Number of CPUs required for this step.
1137
- gpu : int, optional, default None
1138
- Number of GPUs required for this step.
1139
- disk : int, optional, default None
1140
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1141
- memory : int, default 4096
1142
- Memory size (in MB) required for this step.
1143
- shared_memory : int, optional, default None
1144
- The value for the size (in MiB) of the /dev/shm volume for this step.
1145
- This parameter maps to the `--shm-size` option in Docker.
872
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
873
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
874
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
875
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
876
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
877
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
878
+
879
+ temp_dir_root : str, default: None
880
+ The root directory under which `current.model.loaded` will store loaded models
1146
881
  """
1147
882
  ...
1148
883
 
1149
884
  @typing.overload
1150
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
885
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
886
+ ...
887
+
888
+ @typing.overload
889
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
890
+ ...
891
+
892
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1151
893
  """
1152
- Specifies a timeout for your step.
1153
-
1154
- This decorator is useful if this step may hang indefinitely.
894
+ Enables loading / saving of models within a step.
1155
895
 
1156
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1157
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1158
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
896
+ > Examples
897
+ - Saving Models
898
+ ```python
899
+ @model
900
+ @step
901
+ def train(self):
902
+ # current.model.save returns a dictionary reference to the model saved
903
+ self.my_model = current.model.save(
904
+ path_to_my_model,
905
+ label="my_model",
906
+ metadata={
907
+ "epochs": 10,
908
+ "batch-size": 32,
909
+ "learning-rate": 0.001,
910
+ }
911
+ )
912
+ self.next(self.test)
1159
913
 
1160
- Note that all the values specified in parameters are added together so if you specify
1161
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
914
+ @model(load="my_model")
915
+ @step
916
+ def test(self):
917
+ # `current.model.loaded` returns a dictionary of the loaded models
918
+ # where the key is the name of the artifact and the value is the path to the model
919
+ print(os.listdir(current.model.loaded["my_model"]))
920
+ self.next(self.end)
921
+ ```
922
+
923
+ - Loading models
924
+ ```python
925
+ @step
926
+ def train(self):
927
+ # current.model.load returns the path to the model loaded
928
+ checkpoint_path = current.model.load(
929
+ self.checkpoint_key,
930
+ )
931
+ model_path = current.model.load(
932
+ self.model,
933
+ )
934
+ self.next(self.test)
935
+ ```
1162
936
 
1163
937
 
1164
938
  Parameters
1165
939
  ----------
1166
- seconds : int, default 0
1167
- Number of seconds to wait prior to timing out.
1168
- minutes : int, default 0
1169
- Number of minutes to wait prior to timing out.
1170
- hours : int, default 0
1171
- Number of hours to wait prior to timing out.
940
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
941
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
942
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
943
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
944
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
945
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
946
+
947
+ temp_dir_root : str, default: None
948
+ The root directory under which `current.model.loaded` will store loaded models
1172
949
  """
1173
950
  ...
1174
951
 
1175
- @typing.overload
1176
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1177
- ...
1178
-
1179
- @typing.overload
1180
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1181
- ...
1182
-
1183
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
952
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1184
953
  """
1185
- Specifies a timeout for your step.
1186
-
1187
- This decorator is useful if this step may hang indefinitely.
1188
-
1189
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1190
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1191
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1192
-
1193
- Note that all the values specified in parameters are added together so if you specify
1194
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
954
+ Specifies that this step should execute on DGX cloud.
1195
955
 
1196
956
 
1197
957
  Parameters
1198
958
  ----------
1199
- seconds : int, default 0
1200
- Number of seconds to wait prior to timing out.
1201
- minutes : int, default 0
1202
- Number of minutes to wait prior to timing out.
1203
- hours : int, default 0
1204
- Number of hours to wait prior to timing out.
1205
- """
1206
- ...
1207
-
1208
- @typing.overload
1209
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1210
- """
1211
- Decorator prototype for all step decorators. This function gets specialized
1212
- and imported for all decorators types by _import_plugin_decorators().
1213
- """
1214
- ...
1215
-
1216
- @typing.overload
1217
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1218
- ...
1219
-
1220
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1221
- """
1222
- Decorator prototype for all step decorators. This function gets specialized
1223
- and imported for all decorators types by _import_plugin_decorators().
959
+ gpu : int
960
+ Number of GPUs to use.
961
+ gpu_type : str
962
+ Type of Nvidia GPU to use.
963
+ queue_timeout : int
964
+ Time to keep the job in NVCF's queue.
1224
965
  """
1225
966
  ...
1226
967
 
1227
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
968
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1228
969
  """
1229
- S3 Proxy decorator for routing S3 requests through a local proxy service.
970
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
971
+
972
+ User code call
973
+ --------------
974
+ @ollama(
975
+ models=[...],
976
+ ...
977
+ )
978
+
979
+ Valid backend options
980
+ ---------------------
981
+ - 'local': Run as a separate process on the local task machine.
982
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
983
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
984
+
985
+ Valid model options
986
+ -------------------
987
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
1230
988
 
1231
989
 
1232
990
  Parameters
1233
991
  ----------
1234
- integration_name : str, optional
1235
- Name of the S3 proxy integration. If not specified, will use the only
1236
- available S3 proxy integration in the namespace (fails if multiple exist).
1237
- write_mode : str, optional
1238
- The desired behavior during write operations to target (origin) S3 bucket.
1239
- allowed options are:
1240
- "origin-and-cache" -> write to both the target S3 bucket and local object
1241
- storage
1242
- "origin" -> only write to the target S3 bucket
1243
- "cache" -> only write to the object storage service used for caching
1244
- debug : bool, optional
1245
- Enable debug logging for proxy operations.
992
+ models: list[str]
993
+ List of Ollama containers running models in sidecars.
994
+ backend: str
995
+ Determines where and how to run the Ollama process.
996
+ force_pull: bool
997
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
998
+ cache_update_policy: str
999
+ Cache update policy: "auto", "force", or "never".
1000
+ force_cache_update: bool
1001
+ Simple override for "force" cache update policy.
1002
+ debug: bool
1003
+ Whether to turn on verbose debugging logs.
1004
+ circuit_breaker_config: dict
1005
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
1006
+ timeout_config: dict
1007
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
1246
1008
  """
1247
1009
  ...
1248
1010
 
@@ -1305,46 +1067,420 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1305
1067
  """
1306
1068
  ...
1307
1069
 
1308
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1070
+ @typing.overload
1071
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1309
1072
  """
1310
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
1073
+ Decorator prototype for all step decorators. This function gets specialized
1074
+ and imported for all decorators types by _import_plugin_decorators().
1075
+ """
1076
+ ...
1077
+
1078
+ @typing.overload
1079
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1080
+ ...
1081
+
1082
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1083
+ """
1084
+ Decorator prototype for all step decorators. This function gets specialized
1085
+ and imported for all decorators types by _import_plugin_decorators().
1086
+ """
1087
+ ...
1088
+
1089
+ @typing.overload
1090
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1091
+ """
1092
+ Specifies the resources needed when executing this step.
1311
1093
 
1312
- User code call
1313
- --------------
1314
- @ollama(
1315
- models=[...],
1316
- ...
1317
- )
1094
+ Use `@resources` to specify the resource requirements
1095
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1318
1096
 
1319
- Valid backend options
1320
- ---------------------
1321
- - 'local': Run as a separate process on the local task machine.
1322
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
1323
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
1097
+ You can choose the compute layer on the command line by executing e.g.
1098
+ ```
1099
+ python myflow.py run --with batch
1100
+ ```
1101
+ or
1102
+ ```
1103
+ python myflow.py run --with kubernetes
1104
+ ```
1105
+ which executes the flow on the desired system using the
1106
+ requirements specified in `@resources`.
1324
1107
 
1325
- Valid model options
1326
- -------------------
1327
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
1108
+
1109
+ Parameters
1110
+ ----------
1111
+ cpu : int, default 1
1112
+ Number of CPUs required for this step.
1113
+ gpu : int, optional, default None
1114
+ Number of GPUs required for this step.
1115
+ disk : int, optional, default None
1116
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1117
+ memory : int, default 4096
1118
+ Memory size (in MB) required for this step.
1119
+ shared_memory : int, optional, default None
1120
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1121
+ This parameter maps to the `--shm-size` option in Docker.
1122
+ """
1123
+ ...
1124
+
1125
+ @typing.overload
1126
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1127
+ ...
1128
+
1129
+ @typing.overload
1130
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1131
+ ...
1132
+
1133
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1134
+ """
1135
+ Specifies the resources needed when executing this step.
1136
+
1137
+ Use `@resources` to specify the resource requirements
1138
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1139
+
1140
+ You can choose the compute layer on the command line by executing e.g.
1141
+ ```
1142
+ python myflow.py run --with batch
1143
+ ```
1144
+ or
1145
+ ```
1146
+ python myflow.py run --with kubernetes
1147
+ ```
1148
+ which executes the flow on the desired system using the
1149
+ requirements specified in `@resources`.
1328
1150
 
1329
1151
 
1330
1152
  Parameters
1331
1153
  ----------
1332
- models: list[str]
1333
- List of Ollama containers running models in sidecars.
1334
- backend: str
1335
- Determines where and how to run the Ollama process.
1336
- force_pull: bool
1337
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
1338
- cache_update_policy: str
1339
- Cache update policy: "auto", "force", or "never".
1340
- force_cache_update: bool
1341
- Simple override for "force" cache update policy.
1154
+ cpu : int, default 1
1155
+ Number of CPUs required for this step.
1156
+ gpu : int, optional, default None
1157
+ Number of GPUs required for this step.
1158
+ disk : int, optional, default None
1159
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1160
+ memory : int, default 4096
1161
+ Memory size (in MB) required for this step.
1162
+ shared_memory : int, optional, default None
1163
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1164
+ This parameter maps to the `--shm-size` option in Docker.
1165
+ """
1166
+ ...
1167
+
1168
+ @typing.overload
1169
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1170
+ """
1171
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1172
+ It exists to make it easier for users to know that this decorator should only be used with
1173
+ a Neo Cloud like CoreWeave.
1174
+ """
1175
+ ...
1176
+
1177
+ @typing.overload
1178
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1179
+ ...
1180
+
1181
+ def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1182
+ """
1183
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1184
+ It exists to make it easier for users to know that this decorator should only be used with
1185
+ a Neo Cloud like CoreWeave.
1186
+ """
1187
+ ...
1188
+
1189
+ @typing.overload
1190
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1191
+ """
1192
+ Specifies secrets to be retrieved and injected as environment variables prior to
1193
+ the execution of a step.
1194
+
1195
+
1196
+ Parameters
1197
+ ----------
1198
+ sources : List[Union[str, Dict[str, Any]]], default: []
1199
+ List of secret specs, defining how the secrets are to be retrieved
1200
+ role : str, optional, default: None
1201
+ Role to use for fetching secrets
1202
+ """
1203
+ ...
1204
+
1205
+ @typing.overload
1206
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1207
+ ...
1208
+
1209
+ @typing.overload
1210
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1211
+ ...
1212
+
1213
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
1214
+ """
1215
+ Specifies secrets to be retrieved and injected as environment variables prior to
1216
+ the execution of a step.
1217
+
1218
+
1219
+ Parameters
1220
+ ----------
1221
+ sources : List[Union[str, Dict[str, Any]]], default: []
1222
+ List of secret specs, defining how the secrets are to be retrieved
1223
+ role : str, optional, default: None
1224
+ Role to use for fetching secrets
1225
+ """
1226
+ ...
1227
+
1228
+ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1229
+ """
1230
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
1231
+
1232
+ User code call
1233
+ --------------
1234
+ @vllm(
1235
+ model="...",
1236
+ ...
1237
+ )
1238
+
1239
+ Valid backend options
1240
+ ---------------------
1241
+ - 'local': Run as a separate process on the local task machine.
1242
+
1243
+ Valid model options
1244
+ -------------------
1245
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
1246
+
1247
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
1248
+ If you need multiple models, you must create multiple @vllm decorators.
1249
+
1250
+
1251
+ Parameters
1252
+ ----------
1253
+ model: str
1254
+ HuggingFace model identifier to be served by vLLM.
1255
+ backend: str
1256
+ Determines where and how to run the vLLM process.
1257
+ openai_api_server: bool
1258
+ Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
1259
+ Default is False (uses native engine).
1260
+ Set to True for backward compatibility with existing code.
1342
1261
  debug: bool
1343
1262
  Whether to turn on verbose debugging logs.
1344
- circuit_breaker_config: dict
1345
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
1346
- timeout_config: dict
1347
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
1263
+ card_refresh_interval: int
1264
+ Interval in seconds for refreshing the vLLM status card.
1265
+ Only used when openai_api_server=True.
1266
+ max_retries: int
1267
+ Maximum number of retries checking for vLLM server startup.
1268
+ Only used when openai_api_server=True.
1269
+ retry_alert_frequency: int
1270
+ Frequency of alert logs for vLLM server startup retries.
1271
+ Only used when openai_api_server=True.
1272
+ engine_args : dict
1273
+ Additional keyword arguments to pass to the vLLM engine.
1274
+ For example, `tensor_parallel_size=2`.
1275
+ """
1276
+ ...
1277
+
1278
+ @typing.overload
1279
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1280
+ """
1281
+ Specifies that the step will success under all circumstances.
1282
+
1283
+ The decorator will create an optional artifact, specified by `var`, which
1284
+ contains the exception raised. You can use it to detect the presence
1285
+ of errors, indicating that all happy-path artifacts produced by the step
1286
+ are missing.
1287
+
1288
+
1289
+ Parameters
1290
+ ----------
1291
+ var : str, optional, default None
1292
+ Name of the artifact in which to store the caught exception.
1293
+ If not specified, the exception is not stored.
1294
+ print_exception : bool, default True
1295
+ Determines whether or not the exception is printed to
1296
+ stdout when caught.
1297
+ """
1298
+ ...
1299
+
1300
+ @typing.overload
1301
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1302
+ ...
1303
+
1304
+ @typing.overload
1305
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1306
+ ...
1307
+
1308
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1309
+ """
1310
+ Specifies that the step will success under all circumstances.
1311
+
1312
+ The decorator will create an optional artifact, specified by `var`, which
1313
+ contains the exception raised. You can use it to detect the presence
1314
+ of errors, indicating that all happy-path artifacts produced by the step
1315
+ are missing.
1316
+
1317
+
1318
+ Parameters
1319
+ ----------
1320
+ var : str, optional, default None
1321
+ Name of the artifact in which to store the caught exception.
1322
+ If not specified, the exception is not stored.
1323
+ print_exception : bool, default True
1324
+ Determines whether or not the exception is printed to
1325
+ stdout when caught.
1326
+ """
1327
+ ...
1328
+
1329
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1330
+ """
1331
+ S3 Proxy decorator for routing S3 requests through a local proxy service.
1332
+
1333
+
1334
+ Parameters
1335
+ ----------
1336
+ integration_name : str, optional
1337
+ Name of the S3 proxy integration. If not specified, will use the only
1338
+ available S3 proxy integration in the namespace (fails if multiple exist).
1339
+ write_mode : str, optional
1340
+ The desired behavior during write operations to target (origin) S3 bucket.
1341
+ allowed options are:
1342
+ "origin-and-cache" -> write to both the target S3 bucket and local object
1343
+ storage
1344
+ "origin" -> only write to the target S3 bucket
1345
+ "cache" -> only write to the object storage service used for caching
1346
+ debug : bool, optional
1347
+ Enable debug logging for proxy operations.
1348
+ """
1349
+ ...
1350
+
1351
+ @typing.overload
1352
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1353
+ """
1354
+ Specifies the event(s) that this flow depends on.
1355
+
1356
+ ```
1357
+ @trigger(event='foo')
1358
+ ```
1359
+ or
1360
+ ```
1361
+ @trigger(events=['foo', 'bar'])
1362
+ ```
1363
+
1364
+ Additionally, you can specify the parameter mappings
1365
+ to map event payload to Metaflow parameters for the flow.
1366
+ ```
1367
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1368
+ ```
1369
+ or
1370
+ ```
1371
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1372
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1373
+ ```
1374
+
1375
+ 'parameters' can also be a list of strings and tuples like so:
1376
+ ```
1377
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1378
+ ```
1379
+ This is equivalent to:
1380
+ ```
1381
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1382
+ ```
1383
+
1384
+
1385
+ Parameters
1386
+ ----------
1387
+ event : Union[str, Dict[str, Any]], optional, default None
1388
+ Event dependency for this flow.
1389
+ events : List[Union[str, Dict[str, Any]]], default []
1390
+ Events dependency for this flow.
1391
+ options : Dict[str, Any], default {}
1392
+ Backend-specific configuration for tuning eventing behavior.
1393
+ """
1394
+ ...
1395
+
1396
+ @typing.overload
1397
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1398
+ ...
1399
+
1400
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1401
+ """
1402
+ Specifies the event(s) that this flow depends on.
1403
+
1404
+ ```
1405
+ @trigger(event='foo')
1406
+ ```
1407
+ or
1408
+ ```
1409
+ @trigger(events=['foo', 'bar'])
1410
+ ```
1411
+
1412
+ Additionally, you can specify the parameter mappings
1413
+ to map event payload to Metaflow parameters for the flow.
1414
+ ```
1415
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1416
+ ```
1417
+ or
1418
+ ```
1419
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1420
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1421
+ ```
1422
+
1423
+ 'parameters' can also be a list of strings and tuples like so:
1424
+ ```
1425
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1426
+ ```
1427
+ This is equivalent to:
1428
+ ```
1429
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1430
+ ```
1431
+
1432
+
1433
+ Parameters
1434
+ ----------
1435
+ event : Union[str, Dict[str, Any]], optional, default None
1436
+ Event dependency for this flow.
1437
+ events : List[Union[str, Dict[str, Any]]], default []
1438
+ Events dependency for this flow.
1439
+ options : Dict[str, Any], default {}
1440
+ Backend-specific configuration for tuning eventing behavior.
1441
+ """
1442
+ ...
1443
+
1444
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1445
+ """
1446
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1447
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1448
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1449
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1450
+ starts only after all sensors finish.
1451
+
1452
+
1453
+ Parameters
1454
+ ----------
1455
+ timeout : int
1456
+ Time, in seconds before the task times out and fails. (Default: 3600)
1457
+ poke_interval : int
1458
+ Time in seconds that the job should wait in between each try. (Default: 60)
1459
+ mode : str
1460
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1461
+ exponential_backoff : bool
1462
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1463
+ pool : str
1464
+ the slot pool this task should run in,
1465
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1466
+ soft_fail : bool
1467
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1468
+ name : str
1469
+ Name of the sensor on Airflow
1470
+ description : str
1471
+ Description of sensor in the Airflow UI
1472
+ bucket_key : Union[str, List[str]]
1473
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1474
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1475
+ bucket_name : str
1476
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1477
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1478
+ wildcard_match : bool
1479
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1480
+ aws_conn_id : str
1481
+ a reference to the s3 connection on Airflow. (Default: None)
1482
+ verify : bool
1483
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1348
1484
  """
1349
1485
  ...
1350
1486
 
@@ -1399,49 +1535,6 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1399
1535
  """
1400
1536
  ...
1401
1537
 
1402
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1403
- """
1404
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1405
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1406
-
1407
-
1408
- Parameters
1409
- ----------
1410
- timeout : int
1411
- Time, in seconds before the task times out and fails. (Default: 3600)
1412
- poke_interval : int
1413
- Time in seconds that the job should wait in between each try. (Default: 60)
1414
- mode : str
1415
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1416
- exponential_backoff : bool
1417
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1418
- pool : str
1419
- the slot pool this task should run in,
1420
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1421
- soft_fail : bool
1422
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1423
- name : str
1424
- Name of the sensor on Airflow
1425
- description : str
1426
- Description of sensor in the Airflow UI
1427
- external_dag_id : str
1428
- The dag_id that contains the task you want to wait for.
1429
- external_task_ids : List[str]
1430
- The list of task_ids that you want to wait for.
1431
- If None (default value) the sensor waits for the DAG. (Default: None)
1432
- allowed_states : List[str]
1433
- Iterable of allowed states, (Default: ['success'])
1434
- failed_states : List[str]
1435
- Iterable of failed or dis-allowed states. (Default: None)
1436
- execution_delta : datetime.timedelta
1437
- time difference with the previous execution to look at,
1438
- the default is the same logical date as the current task or DAG. (Default: None)
1439
- check_existence: bool
1440
- Set to True to check if the external task exists or check if
1441
- the DAG to wait for exists. (Default: True)
1442
- """
1443
- ...
1444
-
1445
1538
  @typing.overload
1446
1539
  def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1447
1540
  """
@@ -1493,127 +1586,51 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1493
1586
  """
1494
1587
  ...
1495
1588
 
1496
- def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1589
+ @typing.overload
1590
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1497
1591
  """
1498
- Allows setting external datastores to save data for the
1499
- `@checkpoint`/`@model`/`@huggingface_hub` decorators.
1500
-
1501
- This decorator is useful when users wish to save data to a different datastore
1502
- than what is configured in Metaflow. This can be for variety of reasons:
1592
+ Specifies the PyPI packages for all steps of the flow.
1503
1593
 
1504
- 1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
1505
- 2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
1506
- - Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
1507
- 3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
1508
- - Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
1594
+ Use `@pypi_base` to set common packages required by all
1595
+ steps and use `@pypi` to specify step-specific overrides.
1509
1596
 
1510
- Usage:
1597
+ Parameters
1511
1598
  ----------
1599
+ packages : Dict[str, str], default: {}
1600
+ Packages to use for this flow. The key is the name of the package
1601
+ and the value is the version to use.
1602
+ python : str, optional, default: None
1603
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1604
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1605
+ """
1606
+ ...
1607
+
1608
+ @typing.overload
1609
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1610
+ ...
1611
+
1612
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1613
+ """
1614
+ Specifies the PyPI packages for all steps of the flow.
1512
1615
 
1513
- - Using a custom IAM role to access the datastore.
1514
-
1515
- ```python
1516
- @with_artifact_store(
1517
- type="s3",
1518
- config=lambda: {
1519
- "root": "s3://my-bucket-foo/path/to/root",
1520
- "role_arn": ROLE,
1521
- },
1522
- )
1523
- class MyFlow(FlowSpec):
1524
-
1525
- @checkpoint
1526
- @step
1527
- def start(self):
1528
- with open("my_file.txt", "w") as f:
1529
- f.write("Hello, World!")
1530
- self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1531
- self.next(self.end)
1532
-
1533
- ```
1534
-
1535
- - Using credentials to access the s3-compatible datastore.
1536
-
1537
- ```python
1538
- @with_artifact_store(
1539
- type="s3",
1540
- config=lambda: {
1541
- "root": "s3://my-bucket-foo/path/to/root",
1542
- "client_params": {
1543
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1544
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1545
- },
1546
- },
1547
- )
1548
- class MyFlow(FlowSpec):
1549
-
1550
- @checkpoint
1551
- @step
1552
- def start(self):
1553
- with open("my_file.txt", "w") as f:
1554
- f.write("Hello, World!")
1555
- self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1556
- self.next(self.end)
1557
-
1558
- ```
1559
-
1560
- - Accessing objects stored in external datastores after task execution.
1561
-
1562
- ```python
1563
- run = Run("CheckpointsTestsFlow/8992")
1564
- with artifact_store_from(run=run, config={
1565
- "client_params": {
1566
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1567
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1568
- },
1569
- }):
1570
- with Checkpoint() as cp:
1571
- latest = cp.list(
1572
- task=run["start"].task
1573
- )[0]
1574
- print(latest)
1575
- cp.load(
1576
- latest,
1577
- "test-checkpoints"
1578
- )
1616
+ Use `@pypi_base` to set common packages required by all
1617
+ steps and use `@pypi` to specify step-specific overrides.
1579
1618
 
1580
- task = Task("TorchTuneFlow/8484/train/53673")
1581
- with artifact_store_from(run=run, config={
1582
- "client_params": {
1583
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1584
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1585
- },
1586
- }):
1587
- load_model(
1588
- task.data.model_ref,
1589
- "test-models"
1590
- )
1591
- ```
1592
- Parameters:
1619
+ Parameters
1593
1620
  ----------
1594
-
1595
- type: str
1596
- The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1597
-
1598
- config: dict or Callable
1599
- Dictionary of configuration options for the datastore. The following keys are required:
1600
- - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1601
- - example: 's3://bucket-name/path/to/root'
1602
- - example: 'gs://bucket-name/path/to/root'
1603
- - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1604
- - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1605
- - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1606
- - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1621
+ packages : Dict[str, str], default: {}
1622
+ Packages to use for this flow. The key is the name of the package
1623
+ and the value is the version to use.
1624
+ python : str, optional, default: None
1625
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1626
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1607
1627
  """
1608
1628
  ...
1609
1629
 
1610
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1630
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1611
1631
  """
1612
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1613
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1614
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1615
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1616
- starts only after all sensors finish.
1632
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1633
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1617
1634
 
1618
1635
 
1619
1636
  Parameters
@@ -1635,18 +1652,56 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1635
1652
  Name of the sensor on Airflow
1636
1653
  description : str
1637
1654
  Description of sensor in the Airflow UI
1638
- bucket_key : Union[str, List[str]]
1639
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1640
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1641
- bucket_name : str
1642
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1643
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1644
- wildcard_match : bool
1645
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1646
- aws_conn_id : str
1647
- a reference to the s3 connection on Airflow. (Default: None)
1648
- verify : bool
1649
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1655
+ external_dag_id : str
1656
+ The dag_id that contains the task you want to wait for.
1657
+ external_task_ids : List[str]
1658
+ The list of task_ids that you want to wait for.
1659
+ If None (default value) the sensor waits for the DAG. (Default: None)
1660
+ allowed_states : List[str]
1661
+ Iterable of allowed states, (Default: ['success'])
1662
+ failed_states : List[str]
1663
+ Iterable of failed or dis-allowed states. (Default: None)
1664
+ execution_delta : datetime.timedelta
1665
+ time difference with the previous execution to look at,
1666
+ the default is the same logical date as the current task or DAG. (Default: None)
1667
+ check_existence: bool
1668
+ Set to True to check if the external task exists or check if
1669
+ the DAG to wait for exists. (Default: True)
1670
+ """
1671
+ ...
1672
+
1673
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1674
+ """
1675
+ Specifies what flows belong to the same project.
1676
+
1677
+ A project-specific namespace is created for all flows that
1678
+ use the same `@project(name)`.
1679
+
1680
+
1681
+ Parameters
1682
+ ----------
1683
+ name : str
1684
+ Project name. Make sure that the name is unique amongst all
1685
+ projects that use the same production scheduler. The name may
1686
+ contain only lowercase alphanumeric characters and underscores.
1687
+
1688
+ branch : Optional[str], default None
1689
+ The branch to use. If not specified, the branch is set to
1690
+ `user.<username>` unless `production` is set to `True`. This can
1691
+ also be set on the command line using `--branch` as a top-level option.
1692
+ It is an error to specify `branch` in the decorator and on the command line.
1693
+
1694
+ production : bool, default False
1695
+ Whether or not the branch is the production branch. This can also be set on the
1696
+ command line using `--production` as a top-level option. It is an error to specify
1697
+ `production` in the decorator and on the command line.
1698
+ The project branch name will be:
1699
+ - if `branch` is specified:
1700
+ - if `production` is True: `prod.<branch>`
1701
+ - if `production` is False: `test.<branch>`
1702
+ - if `branch` is not specified:
1703
+ - if `production` is True: `prod`
1704
+ - if `production` is False: `user.<username>`
1650
1705
  """
1651
1706
  ...
1652
1707
 
@@ -1751,172 +1806,117 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1751
1806
  """
1752
1807
  ...
1753
1808
 
1754
- @typing.overload
1755
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1809
+ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1756
1810
  """
1757
- Specifies the event(s) that this flow depends on.
1758
-
1759
- ```
1760
- @trigger(event='foo')
1761
- ```
1762
- or
1763
- ```
1764
- @trigger(events=['foo', 'bar'])
1765
- ```
1766
-
1767
- Additionally, you can specify the parameter mappings
1768
- to map event payload to Metaflow parameters for the flow.
1769
- ```
1770
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1771
- ```
1772
- or
1773
- ```
1774
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1775
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1776
- ```
1811
+ Allows setting external datastores to save data for the
1812
+ `@checkpoint`/`@model`/`@huggingface_hub` decorators.
1777
1813
 
1778
- 'parameters' can also be a list of strings and tuples like so:
1779
- ```
1780
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1781
- ```
1782
- This is equivalent to:
1783
- ```
1784
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1785
- ```
1814
+ This decorator is useful when users wish to save data to a different datastore
1815
+ than what is configured in Metaflow. This can be for variety of reasons:
1786
1816
 
1817
+ 1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
1818
+ 2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
1819
+ - Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
1820
+ 3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
1821
+ - Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
1787
1822
 
1788
- Parameters
1823
+ Usage:
1789
1824
  ----------
1790
- event : Union[str, Dict[str, Any]], optional, default None
1791
- Event dependency for this flow.
1792
- events : List[Union[str, Dict[str, Any]]], default []
1793
- Events dependency for this flow.
1794
- options : Dict[str, Any], default {}
1795
- Backend-specific configuration for tuning eventing behavior.
1796
- """
1797
- ...
1798
-
1799
- @typing.overload
1800
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1801
- ...
1802
-
1803
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1804
- """
1805
- Specifies the event(s) that this flow depends on.
1806
-
1807
- ```
1808
- @trigger(event='foo')
1809
- ```
1810
- or
1811
- ```
1812
- @trigger(events=['foo', 'bar'])
1813
- ```
1814
1825
 
1815
- Additionally, you can specify the parameter mappings
1816
- to map event payload to Metaflow parameters for the flow.
1817
- ```
1818
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1819
- ```
1820
- or
1821
- ```
1822
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1823
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1824
- ```
1826
+ - Using a custom IAM role to access the datastore.
1825
1827
 
1826
- 'parameters' can also be a list of strings and tuples like so:
1827
- ```
1828
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1829
- ```
1830
- This is equivalent to:
1831
- ```
1832
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1833
- ```
1828
+ ```python
1829
+ @with_artifact_store(
1830
+ type="s3",
1831
+ config=lambda: {
1832
+ "root": "s3://my-bucket-foo/path/to/root",
1833
+ "role_arn": ROLE,
1834
+ },
1835
+ )
1836
+ class MyFlow(FlowSpec):
1834
1837
 
1838
+ @checkpoint
1839
+ @step
1840
+ def start(self):
1841
+ with open("my_file.txt", "w") as f:
1842
+ f.write("Hello, World!")
1843
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1844
+ self.next(self.end)
1835
1845
 
1836
- Parameters
1837
- ----------
1838
- event : Union[str, Dict[str, Any]], optional, default None
1839
- Event dependency for this flow.
1840
- events : List[Union[str, Dict[str, Any]]], default []
1841
- Events dependency for this flow.
1842
- options : Dict[str, Any], default {}
1843
- Backend-specific configuration for tuning eventing behavior.
1844
- """
1845
- ...
1846
-
1847
- @typing.overload
1848
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1849
- """
1850
- Specifies the PyPI packages for all steps of the flow.
1846
+ ```
1851
1847
 
1852
- Use `@pypi_base` to set common packages required by all
1853
- steps and use `@pypi` to specify step-specific overrides.
1848
+ - Using credentials to access the s3-compatible datastore.
1854
1849
 
1855
- Parameters
1856
- ----------
1857
- packages : Dict[str, str], default: {}
1858
- Packages to use for this flow. The key is the name of the package
1859
- and the value is the version to use.
1860
- python : str, optional, default: None
1861
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1862
- that the version used will correspond to the version of the Python interpreter used to start the run.
1863
- """
1864
- ...
1865
-
1866
- @typing.overload
1867
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1868
- ...
1869
-
1870
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1871
- """
1872
- Specifies the PyPI packages for all steps of the flow.
1850
+ ```python
1851
+ @with_artifact_store(
1852
+ type="s3",
1853
+ config=lambda: {
1854
+ "root": "s3://my-bucket-foo/path/to/root",
1855
+ "client_params": {
1856
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1857
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1858
+ },
1859
+ },
1860
+ )
1861
+ class MyFlow(FlowSpec):
1873
1862
 
1874
- Use `@pypi_base` to set common packages required by all
1875
- steps and use `@pypi` to specify step-specific overrides.
1863
+ @checkpoint
1864
+ @step
1865
+ def start(self):
1866
+ with open("my_file.txt", "w") as f:
1867
+ f.write("Hello, World!")
1868
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1869
+ self.next(self.end)
1876
1870
 
1877
- Parameters
1878
- ----------
1879
- packages : Dict[str, str], default: {}
1880
- Packages to use for this flow. The key is the name of the package
1881
- and the value is the version to use.
1882
- python : str, optional, default: None
1883
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1884
- that the version used will correspond to the version of the Python interpreter used to start the run.
1885
- """
1886
- ...
1887
-
1888
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1889
- """
1890
- Specifies what flows belong to the same project.
1871
+ ```
1891
1872
 
1892
- A project-specific namespace is created for all flows that
1893
- use the same `@project(name)`.
1873
+ - Accessing objects stored in external datastores after task execution.
1894
1874
 
1875
+ ```python
1876
+ run = Run("CheckpointsTestsFlow/8992")
1877
+ with artifact_store_from(run=run, config={
1878
+ "client_params": {
1879
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1880
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1881
+ },
1882
+ }):
1883
+ with Checkpoint() as cp:
1884
+ latest = cp.list(
1885
+ task=run["start"].task
1886
+ )[0]
1887
+ print(latest)
1888
+ cp.load(
1889
+ latest,
1890
+ "test-checkpoints"
1891
+ )
1895
1892
 
1896
- Parameters
1893
+ task = Task("TorchTuneFlow/8484/train/53673")
1894
+ with artifact_store_from(run=run, config={
1895
+ "client_params": {
1896
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1897
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1898
+ },
1899
+ }):
1900
+ load_model(
1901
+ task.data.model_ref,
1902
+ "test-models"
1903
+ )
1904
+ ```
1905
+ Parameters:
1897
1906
  ----------
1898
- name : str
1899
- Project name. Make sure that the name is unique amongst all
1900
- projects that use the same production scheduler. The name may
1901
- contain only lowercase alphanumeric characters and underscores.
1902
1907
 
1903
- branch : Optional[str], default None
1904
- The branch to use. If not specified, the branch is set to
1905
- `user.<username>` unless `production` is set to `True`. This can
1906
- also be set on the command line using `--branch` as a top-level option.
1907
- It is an error to specify `branch` in the decorator and on the command line.
1908
+ type: str
1909
+ The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1908
1910
 
1909
- production : bool, default False
1910
- Whether or not the branch is the production branch. This can also be set on the
1911
- command line using `--production` as a top-level option. It is an error to specify
1912
- `production` in the decorator and on the command line.
1913
- The project branch name will be:
1914
- - if `branch` is specified:
1915
- - if `production` is True: `prod.<branch>`
1916
- - if `production` is False: `test.<branch>`
1917
- - if `branch` is not specified:
1918
- - if `production` is True: `prod`
1919
- - if `production` is False: `user.<username>`
1911
+ config: dict or Callable
1912
+ Dictionary of configuration options for the datastore. The following keys are required:
1913
+ - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1914
+ - example: 's3://bucket-name/path/to/root'
1915
+ - example: 'gs://bucket-name/path/to/root'
1916
+ - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1917
+ - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1918
+ - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1919
+ - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1920
1920
  """
1921
1921
  ...
1922
1922