ob-metaflow-stubs 6.0.10.14__py2.py3-none-any.whl → 6.0.10.16__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow-stubs might be problematic. Click here for more details.

Files changed (266) hide show
  1. metaflow-stubs/__init__.pyi +1035 -1035
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +3 -3
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +5 -5
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +93 -93
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/__init__.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/hf_hub_card.pyi +4 -4
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  64. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  65. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +3 -3
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +3 -3
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +5 -5
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +4 -4
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +5 -5
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +3 -3
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  110. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  111. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  112. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  113. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  114. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  116. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  117. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
  118. metaflow-stubs/multicore_utils.pyi +2 -2
  119. metaflow-stubs/ob_internal.pyi +2 -2
  120. metaflow-stubs/packaging_sys/__init__.pyi +7 -7
  121. metaflow-stubs/packaging_sys/backend.pyi +3 -3
  122. metaflow-stubs/packaging_sys/distribution_support.pyi +3 -3
  123. metaflow-stubs/packaging_sys/tar_backend.pyi +4 -4
  124. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  125. metaflow-stubs/packaging_sys/v1.pyi +3 -3
  126. metaflow-stubs/parameters.pyi +5 -5
  127. metaflow-stubs/plugins/__init__.pyi +10 -10
  128. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  129. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  130. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  131. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  132. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  133. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  134. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  135. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  136. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  137. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  138. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  139. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  140. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  141. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  142. metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
  143. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  144. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  145. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  146. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  147. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  148. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  149. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  150. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  152. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  153. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  154. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  155. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  157. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  158. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  159. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  160. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  161. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  162. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  163. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  164. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  165. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  166. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  167. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  168. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  169. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  170. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  171. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  172. metaflow-stubs/plugins/cards/card_modules/basic.pyi +5 -3
  173. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  174. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  175. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +3 -3
  176. metaflow-stubs/plugins/cards/card_modules/json_viewer.pyi +2 -2
  177. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  178. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  179. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  180. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  181. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  182. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  183. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  184. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  185. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  186. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  187. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  188. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  189. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  190. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  191. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  192. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  193. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  194. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  195. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  196. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  197. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  198. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  199. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  200. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  201. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  202. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  203. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  204. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  205. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  206. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  207. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  208. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  209. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  210. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  211. metaflow-stubs/plugins/optuna/__init__.pyi +2 -2
  212. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  213. metaflow-stubs/plugins/parsers.pyi +2 -2
  214. metaflow-stubs/plugins/perimeters.pyi +2 -2
  215. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  216. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  217. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  218. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  219. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  220. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  221. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  222. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  223. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  224. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  225. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  226. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  227. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  228. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  229. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  230. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  231. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  232. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  233. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  234. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  235. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  236. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  237. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  238. metaflow-stubs/profilers/__init__.pyi +2 -2
  239. metaflow-stubs/pylint_wrapper.pyi +2 -2
  240. metaflow-stubs/runner/__init__.pyi +2 -2
  241. metaflow-stubs/runner/deployer.pyi +34 -34
  242. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  243. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  244. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  245. metaflow-stubs/runner/nbrun.pyi +2 -2
  246. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  247. metaflow-stubs/runner/utils.pyi +3 -3
  248. metaflow-stubs/system/__init__.pyi +2 -2
  249. metaflow-stubs/system/system_logger.pyi +3 -3
  250. metaflow-stubs/system/system_monitor.pyi +2 -2
  251. metaflow-stubs/tagging_util.pyi +2 -2
  252. metaflow-stubs/tuple_util.pyi +2 -2
  253. metaflow-stubs/user_configs/__init__.pyi +2 -2
  254. metaflow-stubs/user_configs/config_options.pyi +3 -3
  255. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  256. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  257. metaflow-stubs/user_decorators/common.pyi +2 -2
  258. metaflow-stubs/user_decorators/mutable_flow.pyi +6 -6
  259. metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
  260. metaflow-stubs/user_decorators/user_flow_decorator.pyi +3 -3
  261. metaflow-stubs/user_decorators/user_step_decorator.pyi +7 -7
  262. {ob_metaflow_stubs-6.0.10.14.dist-info → ob_metaflow_stubs-6.0.10.16.dist-info}/METADATA +1 -1
  263. ob_metaflow_stubs-6.0.10.16.dist-info/RECORD +266 -0
  264. ob_metaflow_stubs-6.0.10.14.dist-info/RECORD +0 -266
  265. {ob_metaflow_stubs-6.0.10.14.dist-info → ob_metaflow_stubs-6.0.10.16.dist-info}/WHEEL +0 -0
  266. {ob_metaflow_stubs-6.0.10.14.dist-info → ob_metaflow_stubs-6.0.10.16.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.18.10.1+obcheckpoint(0.2.8);ob(v1) #
4
- # Generated on 2025-10-08T21:13:44.622359 #
3
+ # MF version: 2.18.11.1+obcheckpoint(0.2.8);ob(v1) #
4
+ # Generated on 2025-10-13T07:07:26.927215 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -40,18 +40,18 @@ from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
42
  from . import cards as cards
43
- from . import events as events
44
43
  from . import tuple_util as tuple_util
45
44
  from . import metaflow_git as metaflow_git
45
+ from . import events as events
46
46
  from . import runner as runner
47
47
  from . import plugins as plugins
48
48
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
49
49
  from . import includefile as includefile
50
50
  from .includefile import IncludeFile as IncludeFile
51
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
52
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
53
51
  from .plugins.parsers import yaml_parser as yaml_parser
52
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
54
53
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
54
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
55
55
  from . import client as client
56
56
  from .client.core import namespace as namespace
57
57
  from .client.core import get_namespace as get_namespace
@@ -169,108 +169,101 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
169
169
  """
170
170
  ...
171
171
 
172
- @typing.overload
173
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
172
+ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
174
173
  """
175
- Specifies a timeout for your step.
174
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
176
175
 
177
- This decorator is useful if this step may hang indefinitely.
176
+ User code call
177
+ --------------
178
+ @vllm(
179
+ model="...",
180
+ ...
181
+ )
178
182
 
179
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
180
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
181
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
183
+ Valid backend options
184
+ ---------------------
185
+ - 'local': Run as a separate process on the local task machine.
182
186
 
183
- Note that all the values specified in parameters are added together so if you specify
184
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
187
+ Valid model options
188
+ -------------------
189
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
190
+
191
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
192
+ If you need multiple models, you must create multiple @vllm decorators.
185
193
 
186
194
 
187
195
  Parameters
188
196
  ----------
189
- seconds : int, default 0
190
- Number of seconds to wait prior to timing out.
191
- minutes : int, default 0
192
- Number of minutes to wait prior to timing out.
193
- hours : int, default 0
194
- Number of hours to wait prior to timing out.
197
+ model: str
198
+ HuggingFace model identifier to be served by vLLM.
199
+ backend: str
200
+ Determines where and how to run the vLLM process.
201
+ openai_api_server: bool
202
+ Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
203
+ Default is False (uses native engine).
204
+ Set to True for backward compatibility with existing code.
205
+ debug: bool
206
+ Whether to turn on verbose debugging logs.
207
+ card_refresh_interval: int
208
+ Interval in seconds for refreshing the vLLM status card.
209
+ Only used when openai_api_server=True.
210
+ max_retries: int
211
+ Maximum number of retries checking for vLLM server startup.
212
+ Only used when openai_api_server=True.
213
+ retry_alert_frequency: int
214
+ Frequency of alert logs for vLLM server startup retries.
215
+ Only used when openai_api_server=True.
216
+ engine_args : dict
217
+ Additional keyword arguments to pass to the vLLM engine.
218
+ For example, `tensor_parallel_size=2`.
195
219
  """
196
220
  ...
197
221
 
198
- @typing.overload
199
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
200
- ...
201
-
202
- @typing.overload
203
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
204
- ...
205
-
206
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
222
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
207
223
  """
208
- Specifies a timeout for your step.
224
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
209
225
 
210
- This decorator is useful if this step may hang indefinitely.
226
+ User code call
227
+ --------------
228
+ @ollama(
229
+ models=[...],
230
+ ...
231
+ )
211
232
 
212
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
213
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
214
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
233
+ Valid backend options
234
+ ---------------------
235
+ - 'local': Run as a separate process on the local task machine.
236
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
237
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
215
238
 
216
- Note that all the values specified in parameters are added together so if you specify
217
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
239
+ Valid model options
240
+ -------------------
241
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
218
242
 
219
243
 
220
244
  Parameters
221
245
  ----------
222
- seconds : int, default 0
223
- Number of seconds to wait prior to timing out.
224
- minutes : int, default 0
225
- Number of minutes to wait prior to timing out.
226
- hours : int, default 0
227
- Number of hours to wait prior to timing out.
228
- """
229
- ...
230
-
231
- @typing.overload
232
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
233
- """
234
- Internal decorator to support Fast bakery
235
- """
236
- ...
237
-
238
- @typing.overload
239
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
240
- ...
241
-
242
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
243
- """
244
- Internal decorator to support Fast bakery
245
- """
246
- ...
247
-
248
- @typing.overload
249
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
250
- """
251
- Decorator prototype for all step decorators. This function gets specialized
252
- and imported for all decorators types by _import_plugin_decorators().
253
- """
254
- ...
255
-
256
- @typing.overload
257
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
258
- ...
259
-
260
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
261
- """
262
- Decorator prototype for all step decorators. This function gets specialized
263
- and imported for all decorators types by _import_plugin_decorators().
246
+ models: list[str]
247
+ List of Ollama containers running models in sidecars.
248
+ backend: str
249
+ Determines where and how to run the Ollama process.
250
+ force_pull: bool
251
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
252
+ cache_update_policy: str
253
+ Cache update policy: "auto", "force", or "never".
254
+ force_cache_update: bool
255
+ Simple override for "force" cache update policy.
256
+ debug: bool
257
+ Whether to turn on verbose debugging logs.
258
+ circuit_breaker_config: dict
259
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
260
+ timeout_config: dict
261
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
264
262
  """
265
263
  ...
266
264
 
267
- def coreweave_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
265
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
268
266
  """
269
- `@coreweave_s3_proxy` is a CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
270
- It exists to make it easier for users to know that this decorator should only be used with
271
- a Neo Cloud like CoreWeave. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
272
-
273
-
274
267
  Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
275
268
  for S3 read and write requests.
276
269
 
@@ -328,200 +321,228 @@ def coreweave_s3_proxy(*, integration_name: typing.Optional[str] = None, write_m
328
321
  """
329
322
  ...
330
323
 
331
- @typing.overload
332
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
324
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
333
325
  """
334
- Specifies environment variables to be set prior to the execution of a step.
326
+ Specifies that this step should execute on Kubernetes.
335
327
 
336
328
 
337
329
  Parameters
338
330
  ----------
339
- vars : Dict[str, str], default {}
340
- Dictionary of environment variables to set.
331
+ cpu : int, default 1
332
+ Number of CPUs required for this step. If `@resources` is
333
+ also present, the maximum value from all decorators is used.
334
+ memory : int, default 4096
335
+ Memory size (in MB) required for this step. If
336
+ `@resources` is also present, the maximum value from all decorators is
337
+ used.
338
+ disk : int, default 10240
339
+ Disk size (in MB) required for this step. If
340
+ `@resources` is also present, the maximum value from all decorators is
341
+ used.
342
+ image : str, optional, default None
343
+ Docker image to use when launching on Kubernetes. If not specified, and
344
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
345
+ not, a default Docker image mapping to the current version of Python is used.
346
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
347
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
348
+ image_pull_secrets: List[str], default []
349
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
350
+ Kubernetes image pull secrets to use when pulling container images
351
+ in Kubernetes.
352
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
353
+ Kubernetes service account to use when launching pod in Kubernetes.
354
+ secrets : List[str], optional, default None
355
+ Kubernetes secrets to use when launching pod in Kubernetes. These
356
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
357
+ in Metaflow configuration.
358
+ node_selector: Union[Dict[str,str], str], optional, default None
359
+ Kubernetes node selector(s) to apply to the pod running the task.
360
+ Can be passed in as a comma separated string of values e.g.
361
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
362
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
363
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
364
+ Kubernetes namespace to use when launching pod in Kubernetes.
365
+ gpu : int, optional, default None
366
+ Number of GPUs required for this step. A value of zero implies that
367
+ the scheduled node should not have GPUs.
368
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
369
+ The vendor of the GPUs to be used for this step.
370
+ tolerations : List[Dict[str,str]], default []
371
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
372
+ Kubernetes tolerations to use when launching pod in Kubernetes.
373
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
374
+ Kubernetes labels to use when launching pod in Kubernetes.
375
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
376
+ Kubernetes annotations to use when launching pod in Kubernetes.
377
+ use_tmpfs : bool, default False
378
+ This enables an explicit tmpfs mount for this step.
379
+ tmpfs_tempdir : bool, default True
380
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
381
+ tmpfs_size : int, optional, default: None
382
+ The value for the size (in MiB) of the tmpfs mount for this step.
383
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
384
+ memory allocated for this step.
385
+ tmpfs_path : str, optional, default /metaflow_temp
386
+ Path to tmpfs mount for this step.
387
+ persistent_volume_claims : Dict[str, str], optional, default None
388
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
389
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
390
+ shared_memory: int, optional
391
+ Shared memory size (in MiB) required for this step
392
+ port: int, optional
393
+ Port number to specify in the Kubernetes job object
394
+ compute_pool : str, optional, default None
395
+ Compute pool to be used for for this step.
396
+ If not specified, any accessible compute pool within the perimeter is used.
397
+ hostname_resolution_timeout: int, default 10 * 60
398
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
399
+ Only applicable when @parallel is used.
400
+ qos: str, default: Burstable
401
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
402
+
403
+ security_context: Dict[str, Any], optional, default None
404
+ Container security context. Applies to the task container. Allows the following keys:
405
+ - privileged: bool, optional, default None
406
+ - allow_privilege_escalation: bool, optional, default None
407
+ - run_as_user: int, optional, default None
408
+ - run_as_group: int, optional, default None
409
+ - run_as_non_root: bool, optional, default None
341
410
  """
342
411
  ...
343
412
 
344
413
  @typing.overload
345
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
414
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
415
+ """
416
+ Decorator prototype for all step decorators. This function gets specialized
417
+ and imported for all decorators types by _import_plugin_decorators().
418
+ """
346
419
  ...
347
420
 
348
421
  @typing.overload
349
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
422
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
350
423
  ...
351
424
 
352
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
425
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
353
426
  """
354
- Specifies environment variables to be set prior to the execution of a step.
355
-
356
-
357
- Parameters
358
- ----------
359
- vars : Dict[str, str], default {}
360
- Dictionary of environment variables to set.
427
+ Decorator prototype for all step decorators. This function gets specialized
428
+ and imported for all decorators types by _import_plugin_decorators().
361
429
  """
362
430
  ...
363
431
 
364
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
432
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
365
433
  """
366
- Specifies that this step should execute on DGX cloud.
434
+ Decorator that helps cache, version, and store models/datasets from the Hugging Face Hub.
367
435
 
436
+ Examples
437
+ --------
368
438
 
369
- Parameters
370
- ----------
371
- gpu : int
372
- Number of GPUs to use.
373
- gpu_type : str
374
- Type of Nvidia GPU to use.
375
- queue_timeout : int
376
- Time to keep the job in NVCF's queue.
377
- """
378
- ...
379
-
380
- @typing.overload
381
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
382
- """
383
- A simple decorator that demonstrates using CardDecoratorInjector
384
- to inject a card and render simple markdown content.
385
- """
386
- ...
387
-
388
- @typing.overload
389
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
390
- ...
391
-
392
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
393
- """
394
- A simple decorator that demonstrates using CardDecoratorInjector
395
- to inject a card and render simple markdown content.
396
- """
397
- ...
398
-
399
- @typing.overload
400
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
401
- """
402
- Specifies the resources needed when executing this step.
439
+ ```python
440
+ # **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
441
+ @huggingface_hub
442
+ @step
443
+ def pull_model_from_huggingface(self):
444
+ # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
445
+ # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
446
+ # with the same `repo_id` in the backend storage, it will not download the model again. The return
447
+ # value of the function is a reference to the model in the backend storage.
448
+ # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
403
449
 
404
- Use `@resources` to specify the resource requirements
405
- independently of the specific compute layer (`@batch`, `@kubernetes`).
450
+ self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
451
+ self.llama_model = current.huggingface_hub.snapshot_download(
452
+ repo_id=self.model_id,
453
+ allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
454
+ )
455
+ self.next(self.train)
406
456
 
407
- You can choose the compute layer on the command line by executing e.g.
408
- ```
409
- python myflow.py run --with batch
410
- ```
411
- or
412
- ```
413
- python myflow.py run --with kubernetes
414
- ```
415
- which executes the flow on the desired system using the
416
- requirements specified in `@resources`.
457
+ # **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
458
+ @huggingface_hub
459
+ @step
460
+ def run_training(self):
461
+ # Temporary directory (auto-cleaned on exit)
462
+ with current.huggingface_hub.load(
463
+ repo_id="google-bert/bert-base-uncased",
464
+ allow_patterns=["*.bin"],
465
+ ) as local_path:
466
+ # Use files under local_path
467
+ train_model(local_path)
468
+ ...
417
469
 
470
+ # **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
418
471
 
419
- Parameters
420
- ----------
421
- cpu : int, default 1
422
- Number of CPUs required for this step.
423
- gpu : int, optional, default None
424
- Number of GPUs required for this step.
425
- disk : int, optional, default None
426
- Disk size (in MB) required for this step. Only applies on Kubernetes.
427
- memory : int, default 4096
428
- Memory size (in MB) required for this step.
429
- shared_memory : int, optional, default None
430
- The value for the size (in MiB) of the /dev/shm volume for this step.
431
- This parameter maps to the `--shm-size` option in Docker.
432
- """
433
- ...
434
-
435
- @typing.overload
436
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
437
- ...
438
-
439
- @typing.overload
440
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
441
- ...
442
-
443
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
444
- """
445
- Specifies the resources needed when executing this step.
472
+ @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
473
+ @step
474
+ def pull_model_from_huggingface(self):
475
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
446
476
 
447
- Use `@resources` to specify the resource requirements
448
- independently of the specific compute layer (`@batch`, `@kubernetes`).
477
+ @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
478
+ @step
479
+ def finetune_model(self):
480
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
481
+ # path_to_model will be /my-directory
449
482
 
450
- You can choose the compute layer on the command line by executing e.g.
451
- ```
452
- python myflow.py run --with batch
453
- ```
454
- or
455
- ```
456
- python myflow.py run --with kubernetes
483
+
484
+ # Takes all the arguments passed to `snapshot_download`
485
+ # except for `local_dir`
486
+ @huggingface_hub(load=[
487
+ {
488
+ "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
489
+ },
490
+ {
491
+ "repo_id": "myorg/mistral-lora",
492
+ "repo_type": "model",
493
+ },
494
+ ])
495
+ @step
496
+ def finetune_model(self):
497
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
498
+ # path_to_model will be /my-directory
457
499
  ```
458
- which executes the flow on the desired system using the
459
- requirements specified in `@resources`.
460
500
 
461
501
 
462
502
  Parameters
463
503
  ----------
464
- cpu : int, default 1
465
- Number of CPUs required for this step.
466
- gpu : int, optional, default None
467
- Number of GPUs required for this step.
468
- disk : int, optional, default None
469
- Disk size (in MB) required for this step. Only applies on Kubernetes.
470
- memory : int, default 4096
471
- Memory size (in MB) required for this step.
472
- shared_memory : int, optional, default None
473
- The value for the size (in MiB) of the /dev/shm volume for this step.
474
- This parameter maps to the `--shm-size` option in Docker.
475
- """
476
- ...
477
-
478
- def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
479
- """
480
- This decorator is used to run vllm APIs as Metaflow task sidecars.
504
+ temp_dir_root : str, optional
505
+ The root directory that will hold the temporary directory where objects will be downloaded.
481
506
 
482
- User code call
483
- --------------
484
- @vllm(
485
- model="...",
486
- ...
487
- )
507
+ cache_scope : str, optional
508
+ The scope of the cache. Can be `checkpoint` / `flow` / `global`.
509
+ - `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
510
+ i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
511
+ Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
488
512
 
489
- Valid backend options
490
- ---------------------
491
- - 'local': Run as a separate process on the local task machine.
513
+ - `flow`: All repos are cached under the flow, regardless of namespace.
514
+ i.e., the cached path is derived solely from the flow name.
515
+ When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
492
516
 
493
- Valid model options
494
- -------------------
495
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
517
+ - `global`: All repos are cached under a globally static path.
518
+ i.e., the base path of the cache is static and all repos are stored under it.
519
+ When to use this mode:
520
+ - All repos from the Hugging Face Hub need to be shared by users across all flow executions.
521
+ - Each caching scope comes with its own trade-offs:
522
+ - `checkpoint`:
523
+ - Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
524
+ - Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
525
+ - `flow`:
526
+ - Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
527
+ - The blast radius of a bad checkpoint is limited to all runs of a particular flow.
528
+ - It doesn't promote cache reuse across flows.
529
+ - `global`:
530
+ - Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
531
+ - It promotes cache reuse across flows.
532
+ - The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
496
533
 
497
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
498
- If you need multiple models, you must create multiple @vllm decorators.
534
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
535
+ The list of repos (models/datasets) to load.
499
536
 
537
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
500
538
 
501
- Parameters
502
- ----------
503
- model: str
504
- HuggingFace model identifier to be served by vLLM.
505
- backend: str
506
- Determines where and how to run the vLLM process.
507
- openai_api_server: bool
508
- Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
509
- Default is False (uses native engine).
510
- Set to True for backward compatibility with existing code.
511
- debug: bool
512
- Whether to turn on verbose debugging logs.
513
- card_refresh_interval: int
514
- Interval in seconds for refreshing the vLLM status card.
515
- Only used when openai_api_server=True.
516
- max_retries: int
517
- Maximum number of retries checking for vLLM server startup.
518
- Only used when openai_api_server=True.
519
- retry_alert_frequency: int
520
- Frequency of alert logs for vLLM server startup retries.
521
- Only used when openai_api_server=True.
522
- engine_args : dict
523
- Additional keyword arguments to pass to the vLLM engine.
524
- For example, `tensor_parallel_size=2`.
539
+ - If repo (model/dataset) is not found in the datastore:
540
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
541
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
542
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
543
+
544
+ - If repo is found in the datastore:
545
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
525
546
  """
526
547
  ...
527
548
 
@@ -590,96 +611,33 @@ def nebius_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode
590
611
  ...
591
612
 
592
613
  @typing.overload
593
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
614
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
594
615
  """
595
- Specifies the number of times the task corresponding
596
- to a step needs to be retried.
597
-
598
- This decorator is useful for handling transient errors, such as networking issues.
599
- If your task contains operations that can't be retried safely, e.g. database updates,
600
- it is advisable to annotate it with `@retry(times=0)`.
601
-
602
- This can be used in conjunction with the `@catch` decorator. The `@catch`
603
- decorator will execute a no-op task after all retries have been exhausted,
604
- ensuring that the flow execution can continue.
605
-
606
-
607
- Parameters
608
- ----------
609
- times : int, default 3
610
- Number of times to retry this task.
611
- minutes_between_retries : int, default 2
612
- Number of minutes between retries.
616
+ Internal decorator to support Fast bakery
613
617
  """
614
618
  ...
615
619
 
616
620
  @typing.overload
617
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
618
- ...
619
-
620
- @typing.overload
621
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
622
- ...
623
-
624
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
625
- """
626
- Specifies the number of times the task corresponding
627
- to a step needs to be retried.
628
-
629
- This decorator is useful for handling transient errors, such as networking issues.
630
- If your task contains operations that can't be retried safely, e.g. database updates,
631
- it is advisable to annotate it with `@retry(times=0)`.
632
-
633
- This can be used in conjunction with the `@catch` decorator. The `@catch`
634
- decorator will execute a no-op task after all retries have been exhausted,
635
- ensuring that the flow execution can continue.
636
-
637
-
638
- Parameters
639
- ----------
640
- times : int, default 3
641
- Number of times to retry this task.
642
- minutes_between_retries : int, default 2
643
- Number of minutes between retries.
644
- """
621
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
645
622
  ...
646
623
 
647
- @typing.overload
648
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
624
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
649
625
  """
650
- Specifies secrets to be retrieved and injected as environment variables prior to
651
- the execution of a step.
652
-
653
-
654
- Parameters
655
- ----------
656
- sources : List[Union[str, Dict[str, Any]]], default: []
657
- List of secret specs, defining how the secrets are to be retrieved
658
- role : str, optional, default: None
659
- Role to use for fetching secrets
626
+ Internal decorator to support Fast bakery
660
627
  """
661
628
  ...
662
629
 
663
- @typing.overload
664
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
665
- ...
666
-
667
- @typing.overload
668
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
669
- ...
670
-
671
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
630
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
672
631
  """
673
- Specifies secrets to be retrieved and injected as environment variables prior to
674
- the execution of a step.
632
+ Specifies that this step should execute on DGX cloud.
675
633
 
676
634
 
677
635
  Parameters
678
636
  ----------
679
- sources : List[Union[str, Dict[str, Any]]], default: []
680
- List of secret specs, defining how the secrets are to be retrieved
681
- role : str, optional, default: None
682
- Role to use for fetching secrets
637
+ gpu : int
638
+ Number of GPUs to use.
639
+ gpu_type : str
640
+ Type of Nvidia GPU to use.
683
641
  """
684
642
  ...
685
643
 
@@ -812,76 +770,126 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
812
770
  """
813
771
  ...
814
772
 
815
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
773
+ @typing.overload
774
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
816
775
  """
817
- Specifies that this step should execute on DGX cloud.
776
+ Specifies the number of times the task corresponding
777
+ to a step needs to be retried.
778
+
779
+ This decorator is useful for handling transient errors, such as networking issues.
780
+ If your task contains operations that can't be retried safely, e.g. database updates,
781
+ it is advisable to annotate it with `@retry(times=0)`.
782
+
783
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
784
+ decorator will execute a no-op task after all retries have been exhausted,
785
+ ensuring that the flow execution can continue.
818
786
 
819
787
 
820
788
  Parameters
821
789
  ----------
822
- gpu : int
823
- Number of GPUs to use.
824
- gpu_type : str
825
- Type of Nvidia GPU to use.
790
+ times : int, default 3
791
+ Number of times to retry this task.
792
+ minutes_between_retries : int, default 2
793
+ Number of minutes between retries.
826
794
  """
827
795
  ...
828
796
 
829
797
  @typing.overload
830
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
798
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
799
+ ...
800
+
801
+ @typing.overload
802
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
803
+ ...
804
+
805
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
831
806
  """
832
- Specifies the Conda environment for the step.
807
+ Specifies the number of times the task corresponding
808
+ to a step needs to be retried.
833
809
 
834
- Information in this decorator will augment any
835
- attributes set in the `@conda_base` flow-level decorator. Hence,
836
- you can use `@conda_base` to set packages required by all
837
- steps and use `@conda` to specify step-specific overrides.
810
+ This decorator is useful for handling transient errors, such as networking issues.
811
+ If your task contains operations that can't be retried safely, e.g. database updates,
812
+ it is advisable to annotate it with `@retry(times=0)`.
813
+
814
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
815
+ decorator will execute a no-op task after all retries have been exhausted,
816
+ ensuring that the flow execution can continue.
838
817
 
839
818
 
840
819
  Parameters
841
820
  ----------
842
- packages : Dict[str, str], default {}
843
- Packages to use for this step. The key is the name of the package
844
- and the value is the version to use.
845
- libraries : Dict[str, str], default {}
846
- Supported for backward compatibility. When used with packages, packages will take precedence.
847
- python : str, optional, default None
848
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
849
- that the version used will correspond to the version of the Python interpreter used to start the run.
850
- disabled : bool, default False
851
- If set to True, disables @conda.
821
+ times : int, default 3
822
+ Number of times to retry this task.
823
+ minutes_between_retries : int, default 2
824
+ Number of minutes between retries.
852
825
  """
853
826
  ...
854
827
 
855
828
  @typing.overload
856
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
829
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
830
+ """
831
+ Creates a human-readable report, a Metaflow Card, after this step completes.
832
+
833
+ Note that you may add multiple `@card` decorators in a step with different parameters.
834
+
835
+
836
+ Parameters
837
+ ----------
838
+ type : str, default 'default'
839
+ Card type.
840
+ id : str, optional, default None
841
+ If multiple cards are present, use this id to identify this card.
842
+ options : Dict[str, Any], default {}
843
+ Options passed to the card. The contents depend on the card type.
844
+ timeout : int, default 45
845
+ Interrupt reporting if it takes more than this many seconds.
846
+ """
857
847
  ...
858
848
 
859
849
  @typing.overload
860
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
850
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
861
851
  ...
862
852
 
863
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
853
+ @typing.overload
854
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
855
+ ...
856
+
857
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
864
858
  """
865
- Specifies the Conda environment for the step.
859
+ Creates a human-readable report, a Metaflow Card, after this step completes.
866
860
 
867
- Information in this decorator will augment any
868
- attributes set in the `@conda_base` flow-level decorator. Hence,
869
- you can use `@conda_base` to set packages required by all
870
- steps and use `@conda` to specify step-specific overrides.
861
+ Note that you may add multiple `@card` decorators in a step with different parameters.
871
862
 
872
863
 
873
864
  Parameters
874
865
  ----------
875
- packages : Dict[str, str], default {}
876
- Packages to use for this step. The key is the name of the package
877
- and the value is the version to use.
878
- libraries : Dict[str, str], default {}
879
- Supported for backward compatibility. When used with packages, packages will take precedence.
880
- python : str, optional, default None
881
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
882
- that the version used will correspond to the version of the Python interpreter used to start the run.
883
- disabled : bool, default False
884
- If set to True, disables @conda.
866
+ type : str, default 'default'
867
+ Card type.
868
+ id : str, optional, default None
869
+ If multiple cards are present, use this id to identify this card.
870
+ options : Dict[str, Any], default {}
871
+ Options passed to the card. The contents depend on the card type.
872
+ timeout : int, default 45
873
+ Interrupt reporting if it takes more than this many seconds.
874
+ """
875
+ ...
876
+
877
+ @typing.overload
878
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
879
+ """
880
+ Decorator prototype for all step decorators. This function gets specialized
881
+ and imported for all decorators types by _import_plugin_decorators().
882
+ """
883
+ ...
884
+
885
+ @typing.overload
886
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
887
+ ...
888
+
889
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
890
+ """
891
+ Decorator prototype for all step decorators. This function gets specialized
892
+ and imported for all decorators types by _import_plugin_decorators().
885
893
  """
886
894
  ...
887
895
 
@@ -1033,309 +1041,283 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
1033
1041
  ...
1034
1042
 
1035
1043
  @typing.overload
1036
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1044
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1037
1045
  """
1038
- Specifies that the step will success under all circumstances.
1046
+ Specifies a timeout for your step.
1039
1047
 
1040
- The decorator will create an optional artifact, specified by `var`, which
1041
- contains the exception raised. You can use it to detect the presence
1042
- of errors, indicating that all happy-path artifacts produced by the step
1043
- are missing.
1048
+ This decorator is useful if this step may hang indefinitely.
1049
+
1050
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1051
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1052
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1053
+
1054
+ Note that all the values specified in parameters are added together so if you specify
1055
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1044
1056
 
1045
1057
 
1046
1058
  Parameters
1047
1059
  ----------
1048
- var : str, optional, default None
1049
- Name of the artifact in which to store the caught exception.
1050
- If not specified, the exception is not stored.
1051
- print_exception : bool, default True
1052
- Determines whether or not the exception is printed to
1053
- stdout when caught.
1060
+ seconds : int, default 0
1061
+ Number of seconds to wait prior to timing out.
1062
+ minutes : int, default 0
1063
+ Number of minutes to wait prior to timing out.
1064
+ hours : int, default 0
1065
+ Number of hours to wait prior to timing out.
1054
1066
  """
1055
1067
  ...
1056
1068
 
1057
1069
  @typing.overload
1058
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1070
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1059
1071
  ...
1060
1072
 
1061
1073
  @typing.overload
1062
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1074
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1063
1075
  ...
1064
1076
 
1065
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1077
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1066
1078
  """
1067
- Specifies that the step will success under all circumstances.
1079
+ Specifies a timeout for your step.
1068
1080
 
1069
- The decorator will create an optional artifact, specified by `var`, which
1070
- contains the exception raised. You can use it to detect the presence
1071
- of errors, indicating that all happy-path artifacts produced by the step
1072
- are missing.
1081
+ This decorator is useful if this step may hang indefinitely.
1082
+
1083
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1084
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1085
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1086
+
1087
+ Note that all the values specified in parameters are added together so if you specify
1088
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1073
1089
 
1074
1090
 
1075
1091
  Parameters
1076
1092
  ----------
1077
- var : str, optional, default None
1078
- Name of the artifact in which to store the caught exception.
1079
- If not specified, the exception is not stored.
1080
- print_exception : bool, default True
1081
- Determines whether or not the exception is printed to
1082
- stdout when caught.
1083
- """
1084
- ...
1085
-
1086
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1087
- """
1088
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
1089
-
1090
- User code call
1091
- --------------
1092
- @ollama(
1093
- models=[...],
1094
- ...
1095
- )
1096
-
1097
- Valid backend options
1098
- ---------------------
1099
- - 'local': Run as a separate process on the local task machine.
1100
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
1101
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
1102
-
1103
- Valid model options
1104
- -------------------
1105
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
1106
-
1107
-
1108
- Parameters
1109
- ----------
1110
- models: list[str]
1111
- List of Ollama containers running models in sidecars.
1112
- backend: str
1113
- Determines where and how to run the Ollama process.
1114
- force_pull: bool
1115
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
1116
- cache_update_policy: str
1117
- Cache update policy: "auto", "force", or "never".
1118
- force_cache_update: bool
1119
- Simple override for "force" cache update policy.
1120
- debug: bool
1121
- Whether to turn on verbose debugging logs.
1122
- circuit_breaker_config: dict
1123
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
1124
- timeout_config: dict
1125
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
1126
- """
1127
- ...
1128
-
1129
- @typing.overload
1130
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1131
- """
1132
- Decorator prototype for all step decorators. This function gets specialized
1133
- and imported for all decorators types by _import_plugin_decorators().
1134
- """
1135
- ...
1136
-
1137
- @typing.overload
1138
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1139
- ...
1140
-
1141
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1142
- """
1143
- Decorator prototype for all step decorators. This function gets specialized
1144
- and imported for all decorators types by _import_plugin_decorators().
1093
+ seconds : int, default 0
1094
+ Number of seconds to wait prior to timing out.
1095
+ minutes : int, default 0
1096
+ Number of minutes to wait prior to timing out.
1097
+ hours : int, default 0
1098
+ Number of hours to wait prior to timing out.
1145
1099
  """
1146
1100
  ...
1147
1101
 
1148
1102
  @typing.overload
1149
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1103
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1150
1104
  """
1151
- Specifies the PyPI packages for the step.
1105
+ Specifies the Conda environment for the step.
1152
1106
 
1153
1107
  Information in this decorator will augment any
1154
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1155
- you can use `@pypi_base` to set packages required by all
1156
- steps and use `@pypi` to specify step-specific overrides.
1108
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1109
+ you can use `@conda_base` to set packages required by all
1110
+ steps and use `@conda` to specify step-specific overrides.
1157
1111
 
1158
1112
 
1159
1113
  Parameters
1160
1114
  ----------
1161
- packages : Dict[str, str], default: {}
1115
+ packages : Dict[str, str], default {}
1162
1116
  Packages to use for this step. The key is the name of the package
1163
1117
  and the value is the version to use.
1164
- python : str, optional, default: None
1118
+ libraries : Dict[str, str], default {}
1119
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1120
+ python : str, optional, default None
1165
1121
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1166
1122
  that the version used will correspond to the version of the Python interpreter used to start the run.
1123
+ disabled : bool, default False
1124
+ If set to True, disables @conda.
1167
1125
  """
1168
1126
  ...
1169
1127
 
1170
1128
  @typing.overload
1171
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1129
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1172
1130
  ...
1173
1131
 
1174
1132
  @typing.overload
1175
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1133
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1176
1134
  ...
1177
1135
 
1178
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1136
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1179
1137
  """
1180
- Specifies the PyPI packages for the step.
1138
+ Specifies the Conda environment for the step.
1181
1139
 
1182
1140
  Information in this decorator will augment any
1183
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1184
- you can use `@pypi_base` to set packages required by all
1185
- steps and use `@pypi` to specify step-specific overrides.
1141
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1142
+ you can use `@conda_base` to set packages required by all
1143
+ steps and use `@conda` to specify step-specific overrides.
1186
1144
 
1187
1145
 
1188
1146
  Parameters
1189
1147
  ----------
1190
- packages : Dict[str, str], default: {}
1148
+ packages : Dict[str, str], default {}
1191
1149
  Packages to use for this step. The key is the name of the package
1192
1150
  and the value is the version to use.
1193
- python : str, optional, default: None
1151
+ libraries : Dict[str, str], default {}
1152
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1153
+ python : str, optional, default None
1194
1154
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1195
1155
  that the version used will correspond to the version of the Python interpreter used to start the run.
1196
- """
1197
- ...
1198
-
1199
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1200
- """
1201
- Specifies that this step should execute on Kubernetes.
1202
-
1203
-
1204
- Parameters
1205
- ----------
1206
- cpu : int, default 1
1207
- Number of CPUs required for this step. If `@resources` is
1208
- also present, the maximum value from all decorators is used.
1209
- memory : int, default 4096
1210
- Memory size (in MB) required for this step. If
1211
- `@resources` is also present, the maximum value from all decorators is
1212
- used.
1213
- disk : int, default 10240
1214
- Disk size (in MB) required for this step. If
1215
- `@resources` is also present, the maximum value from all decorators is
1216
- used.
1217
- image : str, optional, default None
1218
- Docker image to use when launching on Kubernetes. If not specified, and
1219
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1220
- not, a default Docker image mapping to the current version of Python is used.
1221
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1222
- If given, the imagePullPolicy to be applied to the Docker image of the step.
1223
- image_pull_secrets: List[str], default []
1224
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
1225
- Kubernetes image pull secrets to use when pulling container images
1226
- in Kubernetes.
1227
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1228
- Kubernetes service account to use when launching pod in Kubernetes.
1229
- secrets : List[str], optional, default None
1230
- Kubernetes secrets to use when launching pod in Kubernetes. These
1231
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1232
- in Metaflow configuration.
1233
- node_selector: Union[Dict[str,str], str], optional, default None
1234
- Kubernetes node selector(s) to apply to the pod running the task.
1235
- Can be passed in as a comma separated string of values e.g.
1236
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
1237
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
1238
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1239
- Kubernetes namespace to use when launching pod in Kubernetes.
1240
- gpu : int, optional, default None
1241
- Number of GPUs required for this step. A value of zero implies that
1242
- the scheduled node should not have GPUs.
1243
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1244
- The vendor of the GPUs to be used for this step.
1245
- tolerations : List[Dict[str,str]], default []
1246
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1247
- Kubernetes tolerations to use when launching pod in Kubernetes.
1248
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
1249
- Kubernetes labels to use when launching pod in Kubernetes.
1250
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
1251
- Kubernetes annotations to use when launching pod in Kubernetes.
1252
- use_tmpfs : bool, default False
1253
- This enables an explicit tmpfs mount for this step.
1254
- tmpfs_tempdir : bool, default True
1255
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1256
- tmpfs_size : int, optional, default: None
1257
- The value for the size (in MiB) of the tmpfs mount for this step.
1258
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1259
- memory allocated for this step.
1260
- tmpfs_path : str, optional, default /metaflow_temp
1261
- Path to tmpfs mount for this step.
1262
- persistent_volume_claims : Dict[str, str], optional, default None
1263
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1264
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1265
- shared_memory: int, optional
1266
- Shared memory size (in MiB) required for this step
1267
- port: int, optional
1268
- Port number to specify in the Kubernetes job object
1269
- compute_pool : str, optional, default None
1270
- Compute pool to be used for for this step.
1271
- If not specified, any accessible compute pool within the perimeter is used.
1272
- hostname_resolution_timeout: int, default 10 * 60
1273
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
1274
- Only applicable when @parallel is used.
1275
- qos: str, default: Burstable
1276
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
1277
-
1278
- security_context: Dict[str, Any], optional, default None
1279
- Container security context. Applies to the task container. Allows the following keys:
1280
- - privileged: bool, optional, default None
1281
- - allow_privilege_escalation: bool, optional, default None
1282
- - run_as_user: int, optional, default None
1283
- - run_as_group: int, optional, default None
1284
- - run_as_non_root: bool, optional, default None
1156
+ disabled : bool, default False
1157
+ If set to True, disables @conda.
1285
1158
  """
1286
1159
  ...
1287
1160
 
1288
1161
  @typing.overload
1289
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1162
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1290
1163
  """
1291
- Creates a human-readable report, a Metaflow Card, after this step completes.
1164
+ Specifies the resources needed when executing this step.
1292
1165
 
1293
- Note that you may add multiple `@card` decorators in a step with different parameters.
1166
+ Use `@resources` to specify the resource requirements
1167
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1168
+
1169
+ You can choose the compute layer on the command line by executing e.g.
1170
+ ```
1171
+ python myflow.py run --with batch
1172
+ ```
1173
+ or
1174
+ ```
1175
+ python myflow.py run --with kubernetes
1176
+ ```
1177
+ which executes the flow on the desired system using the
1178
+ requirements specified in `@resources`.
1294
1179
 
1295
1180
 
1296
1181
  Parameters
1297
1182
  ----------
1298
- type : str, default 'default'
1299
- Card type.
1300
- id : str, optional, default None
1301
- If multiple cards are present, use this id to identify this card.
1302
- options : Dict[str, Any], default {}
1303
- Options passed to the card. The contents depend on the card type.
1304
- timeout : int, default 45
1305
- Interrupt reporting if it takes more than this many seconds.
1183
+ cpu : int, default 1
1184
+ Number of CPUs required for this step.
1185
+ gpu : int, optional, default None
1186
+ Number of GPUs required for this step.
1187
+ disk : int, optional, default None
1188
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1189
+ memory : int, default 4096
1190
+ Memory size (in MB) required for this step.
1191
+ shared_memory : int, optional, default None
1192
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1193
+ This parameter maps to the `--shm-size` option in Docker.
1306
1194
  """
1307
1195
  ...
1308
1196
 
1309
1197
  @typing.overload
1310
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1198
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1311
1199
  ...
1312
1200
 
1313
1201
  @typing.overload
1314
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1202
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1315
1203
  ...
1316
1204
 
1317
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1205
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1318
1206
  """
1319
- Creates a human-readable report, a Metaflow Card, after this step completes.
1207
+ Specifies the resources needed when executing this step.
1320
1208
 
1321
- Note that you may add multiple `@card` decorators in a step with different parameters.
1209
+ Use `@resources` to specify the resource requirements
1210
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1211
+
1212
+ You can choose the compute layer on the command line by executing e.g.
1213
+ ```
1214
+ python myflow.py run --with batch
1215
+ ```
1216
+ or
1217
+ ```
1218
+ python myflow.py run --with kubernetes
1219
+ ```
1220
+ which executes the flow on the desired system using the
1221
+ requirements specified in `@resources`.
1322
1222
 
1323
1223
 
1324
1224
  Parameters
1325
1225
  ----------
1326
- type : str, default 'default'
1327
- Card type.
1328
- id : str, optional, default None
1329
- If multiple cards are present, use this id to identify this card.
1330
- options : Dict[str, Any], default {}
1331
- Options passed to the card. The contents depend on the card type.
1332
- timeout : int, default 45
1333
- Interrupt reporting if it takes more than this many seconds.
1226
+ cpu : int, default 1
1227
+ Number of CPUs required for this step.
1228
+ gpu : int, optional, default None
1229
+ Number of GPUs required for this step.
1230
+ disk : int, optional, default None
1231
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1232
+ memory : int, default 4096
1233
+ Memory size (in MB) required for this step.
1234
+ shared_memory : int, optional, default None
1235
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1236
+ This parameter maps to the `--shm-size` option in Docker.
1334
1237
  """
1335
1238
  ...
1336
1239
 
1337
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1240
+ @typing.overload
1241
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1242
+ """
1243
+ A simple decorator that demonstrates using CardDecoratorInjector
1244
+ to inject a card and render simple markdown content.
1245
+ """
1246
+ ...
1247
+
1248
+ @typing.overload
1249
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1250
+ ...
1251
+
1252
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1253
+ """
1254
+ A simple decorator that demonstrates using CardDecoratorInjector
1255
+ to inject a card and render simple markdown content.
1256
+ """
1257
+ ...
1258
+
1259
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1260
+ """
1261
+ Specifies that this step should execute on DGX cloud.
1262
+
1263
+
1264
+ Parameters
1265
+ ----------
1266
+ gpu : int
1267
+ Number of GPUs to use.
1268
+ gpu_type : str
1269
+ Type of Nvidia GPU to use.
1270
+ queue_timeout : int
1271
+ Time to keep the job in NVCF's queue.
1272
+ """
1273
+ ...
1274
+
1275
+ @typing.overload
1276
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1277
+ """
1278
+ Specifies secrets to be retrieved and injected as environment variables prior to
1279
+ the execution of a step.
1280
+
1281
+
1282
+ Parameters
1283
+ ----------
1284
+ sources : List[Union[str, Dict[str, Any]]], default: []
1285
+ List of secret specs, defining how the secrets are to be retrieved
1286
+ role : str, optional, default: None
1287
+ Role to use for fetching secrets
1288
+ """
1289
+ ...
1290
+
1291
+ @typing.overload
1292
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1293
+ ...
1294
+
1295
+ @typing.overload
1296
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1297
+ ...
1298
+
1299
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
1338
1300
  """
1301
+ Specifies secrets to be retrieved and injected as environment variables prior to
1302
+ the execution of a step.
1303
+
1304
+
1305
+ Parameters
1306
+ ----------
1307
+ sources : List[Union[str, Dict[str, Any]]], default: []
1308
+ List of secret specs, defining how the secrets are to be retrieved
1309
+ role : str, optional, default: None
1310
+ Role to use for fetching secrets
1311
+ """
1312
+ ...
1313
+
1314
+ def coreweave_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1315
+ """
1316
+ `@coreweave_s3_proxy` is a CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1317
+ It exists to make it easier for users to know that this decorator should only be used with
1318
+ a Neo Cloud like CoreWeave. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
1319
+
1320
+
1339
1321
  Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
1340
1322
  for S3 read and write requests.
1341
1323
 
@@ -1393,221 +1375,336 @@ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typin
1393
1375
  """
1394
1376
  ...
1395
1377
 
1396
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1378
+ @typing.overload
1379
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1397
1380
  """
1398
- Decorator that helps cache, version, and store models/datasets from the Hugging Face Hub.
1399
-
1400
- Examples
1401
- --------
1381
+ Specifies environment variables to be set prior to the execution of a step.
1402
1382
 
1403
- ```python
1404
- # **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
1405
- @huggingface_hub
1406
- @step
1407
- def pull_model_from_huggingface(self):
1408
- # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
1409
- # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
1410
- # with the same `repo_id` in the backend storage, it will not download the model again. The return
1411
- # value of the function is a reference to the model in the backend storage.
1412
- # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
1413
1383
 
1414
- self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
1415
- self.llama_model = current.huggingface_hub.snapshot_download(
1416
- repo_id=self.model_id,
1417
- allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
1418
- )
1419
- self.next(self.train)
1384
+ Parameters
1385
+ ----------
1386
+ vars : Dict[str, str], default {}
1387
+ Dictionary of environment variables to set.
1388
+ """
1389
+ ...
1390
+
1391
+ @typing.overload
1392
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1393
+ ...
1394
+
1395
+ @typing.overload
1396
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1397
+ ...
1398
+
1399
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1400
+ """
1401
+ Specifies environment variables to be set prior to the execution of a step.
1420
1402
 
1421
- # **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
1422
- @huggingface_hub
1423
- @step
1424
- def run_training(self):
1425
- # Temporary directory (auto-cleaned on exit)
1426
- with current.huggingface_hub.load(
1427
- repo_id="google-bert/bert-base-uncased",
1428
- allow_patterns=["*.bin"],
1429
- ) as local_path:
1430
- # Use files under local_path
1431
- train_model(local_path)
1432
- ...
1433
1403
 
1434
- # **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
1404
+ Parameters
1405
+ ----------
1406
+ vars : Dict[str, str], default {}
1407
+ Dictionary of environment variables to set.
1408
+ """
1409
+ ...
1410
+
1411
+ @typing.overload
1412
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1413
+ """
1414
+ Specifies the PyPI packages for the step.
1435
1415
 
1436
- @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
1437
- @step
1438
- def pull_model_from_huggingface(self):
1439
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
1416
+ Information in this decorator will augment any
1417
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1418
+ you can use `@pypi_base` to set packages required by all
1419
+ steps and use `@pypi` to specify step-specific overrides.
1440
1420
 
1441
- @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
1442
- @step
1443
- def finetune_model(self):
1444
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
1445
- # path_to_model will be /my-directory
1446
1421
 
1422
+ Parameters
1423
+ ----------
1424
+ packages : Dict[str, str], default: {}
1425
+ Packages to use for this step. The key is the name of the package
1426
+ and the value is the version to use.
1427
+ python : str, optional, default: None
1428
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1429
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1430
+ """
1431
+ ...
1432
+
1433
+ @typing.overload
1434
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1435
+ ...
1436
+
1437
+ @typing.overload
1438
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1439
+ ...
1440
+
1441
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1442
+ """
1443
+ Specifies the PyPI packages for the step.
1447
1444
 
1448
- # Takes all the arguments passed to `snapshot_download`
1449
- # except for `local_dir`
1450
- @huggingface_hub(load=[
1451
- {
1452
- "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
1453
- },
1454
- {
1455
- "repo_id": "myorg/mistral-lora",
1456
- "repo_type": "model",
1457
- },
1458
- ])
1459
- @step
1460
- def finetune_model(self):
1461
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
1462
- # path_to_model will be /my-directory
1463
- ```
1445
+ Information in this decorator will augment any
1446
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1447
+ you can use `@pypi_base` to set packages required by all
1448
+ steps and use `@pypi` to specify step-specific overrides.
1464
1449
 
1465
1450
 
1466
1451
  Parameters
1467
1452
  ----------
1468
- temp_dir_root : str, optional
1469
- The root directory that will hold the temporary directory where objects will be downloaded.
1453
+ packages : Dict[str, str], default: {}
1454
+ Packages to use for this step. The key is the name of the package
1455
+ and the value is the version to use.
1456
+ python : str, optional, default: None
1457
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1458
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1459
+ """
1460
+ ...
1461
+
1462
+ @typing.overload
1463
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1464
+ """
1465
+ Specifies that the step will success under all circumstances.
1470
1466
 
1471
- cache_scope : str, optional
1472
- The scope of the cache. Can be `checkpoint` / `flow` / `global`.
1473
- - `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
1474
- i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
1475
- Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
1467
+ The decorator will create an optional artifact, specified by `var`, which
1468
+ contains the exception raised. You can use it to detect the presence
1469
+ of errors, indicating that all happy-path artifacts produced by the step
1470
+ are missing.
1476
1471
 
1477
- - `flow`: All repos are cached under the flow, regardless of namespace.
1478
- i.e., the cached path is derived solely from the flow name.
1479
- When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
1480
1472
 
1481
- - `global`: All repos are cached under a globally static path.
1482
- i.e., the base path of the cache is static and all repos are stored under it.
1483
- When to use this mode:
1484
- - All repos from the Hugging Face Hub need to be shared by users across all flow executions.
1485
- - Each caching scope comes with its own trade-offs:
1486
- - `checkpoint`:
1487
- - Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
1488
- - Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
1489
- - `flow`:
1490
- - Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
1491
- - The blast radius of a bad checkpoint is limited to all runs of a particular flow.
1492
- - It doesn't promote cache reuse across flows.
1493
- - `global`:
1494
- - Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
1495
- - It promotes cache reuse across flows.
1496
- - The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
1497
-
1498
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
1499
- The list of repos (models/datasets) to load.
1473
+ Parameters
1474
+ ----------
1475
+ var : str, optional, default None
1476
+ Name of the artifact in which to store the caught exception.
1477
+ If not specified, the exception is not stored.
1478
+ print_exception : bool, default True
1479
+ Determines whether or not the exception is printed to
1480
+ stdout when caught.
1481
+ """
1482
+ ...
1483
+
1484
+ @typing.overload
1485
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1486
+ ...
1487
+
1488
+ @typing.overload
1489
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1490
+ ...
1491
+
1492
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1493
+ """
1494
+ Specifies that the step will success under all circumstances.
1500
1495
 
1501
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
1496
+ The decorator will create an optional artifact, specified by `var`, which
1497
+ contains the exception raised. You can use it to detect the presence
1498
+ of errors, indicating that all happy-path artifacts produced by the step
1499
+ are missing.
1502
1500
 
1503
- - If repo (model/dataset) is not found in the datastore:
1504
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
1505
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
1506
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
1507
1501
 
1508
- - If repo is found in the datastore:
1509
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
1502
+ Parameters
1503
+ ----------
1504
+ var : str, optional, default None
1505
+ Name of the artifact in which to store the caught exception.
1506
+ If not specified, the exception is not stored.
1507
+ print_exception : bool, default True
1508
+ Determines whether or not the exception is printed to
1509
+ stdout when caught.
1510
1510
  """
1511
1511
  ...
1512
1512
 
1513
- @typing.overload
1514
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1513
+ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1515
1514
  """
1516
- Specifies the flow(s) that this flow depends on.
1515
+ Allows setting external datastores to save data for the
1516
+ `@checkpoint`/`@model`/`@huggingface_hub` decorators.
1517
1517
 
1518
- ```
1519
- @trigger_on_finish(flow='FooFlow')
1520
- ```
1521
- or
1522
- ```
1523
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1524
- ```
1525
- This decorator respects the @project decorator and triggers the flow
1526
- when upstream runs within the same namespace complete successfully
1518
+ This decorator is useful when users wish to save data to a different datastore
1519
+ than what is configured in Metaflow. This can be for variety of reasons:
1527
1520
 
1528
- Additionally, you can specify project aware upstream flow dependencies
1529
- by specifying the fully qualified project_flow_name.
1530
- ```
1531
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1532
- ```
1533
- or
1534
- ```
1535
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1536
- ```
1521
+ 1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
1522
+ 2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
1523
+ - Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
1524
+ 3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
1525
+ - Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
1537
1526
 
1538
- You can also specify just the project or project branch (other values will be
1539
- inferred from the current project or project branch):
1540
- ```
1541
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1542
- ```
1527
+ Usage:
1528
+ ----------
1543
1529
 
1544
- Note that `branch` is typically one of:
1545
- - `prod`
1546
- - `user.bob`
1547
- - `test.my_experiment`
1548
- - `prod.staging`
1530
+ - Using a custom IAM role to access the datastore.
1531
+
1532
+ ```python
1533
+ @with_artifact_store(
1534
+ type="s3",
1535
+ config=lambda: {
1536
+ "root": "s3://my-bucket-foo/path/to/root",
1537
+ "role_arn": ROLE,
1538
+ },
1539
+ )
1540
+ class MyFlow(FlowSpec):
1549
1541
 
1542
+ @checkpoint
1543
+ @step
1544
+ def start(self):
1545
+ with open("my_file.txt", "w") as f:
1546
+ f.write("Hello, World!")
1547
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1548
+ self.next(self.end)
1550
1549
 
1551
- Parameters
1550
+ ```
1551
+
1552
+ - Using credentials to access the s3-compatible datastore.
1553
+
1554
+ ```python
1555
+ @with_artifact_store(
1556
+ type="s3",
1557
+ config=lambda: {
1558
+ "root": "s3://my-bucket-foo/path/to/root",
1559
+ "client_params": {
1560
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1561
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1562
+ },
1563
+ },
1564
+ )
1565
+ class MyFlow(FlowSpec):
1566
+
1567
+ @checkpoint
1568
+ @step
1569
+ def start(self):
1570
+ with open("my_file.txt", "w") as f:
1571
+ f.write("Hello, World!")
1572
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1573
+ self.next(self.end)
1574
+
1575
+ ```
1576
+
1577
+ - Accessing objects stored in external datastores after task execution.
1578
+
1579
+ ```python
1580
+ run = Run("CheckpointsTestsFlow/8992")
1581
+ with artifact_store_from(run=run, config={
1582
+ "client_params": {
1583
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1584
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1585
+ },
1586
+ }):
1587
+ with Checkpoint() as cp:
1588
+ latest = cp.list(
1589
+ task=run["start"].task
1590
+ )[0]
1591
+ print(latest)
1592
+ cp.load(
1593
+ latest,
1594
+ "test-checkpoints"
1595
+ )
1596
+
1597
+ task = Task("TorchTuneFlow/8484/train/53673")
1598
+ with artifact_store_from(run=run, config={
1599
+ "client_params": {
1600
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1601
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1602
+ },
1603
+ }):
1604
+ load_model(
1605
+ task.data.model_ref,
1606
+ "test-models"
1607
+ )
1608
+ ```
1609
+ Parameters:
1552
1610
  ----------
1553
- flow : Union[str, Dict[str, str]], optional, default None
1554
- Upstream flow dependency for this flow.
1555
- flows : List[Union[str, Dict[str, str]]], default []
1556
- Upstream flow dependencies for this flow.
1557
- options : Dict[str, Any], default {}
1558
- Backend-specific configuration for tuning eventing behavior.
1611
+
1612
+ type: str
1613
+ The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1614
+
1615
+ config: dict or Callable
1616
+ Dictionary of configuration options for the datastore. The following keys are required:
1617
+ - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1618
+ - example: 's3://bucket-name/path/to/root'
1619
+ - example: 'gs://bucket-name/path/to/root'
1620
+ - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1621
+ - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1622
+ - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1623
+ - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1559
1624
  """
1560
1625
  ...
1561
1626
 
1562
- @typing.overload
1563
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1564
- ...
1565
-
1566
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1627
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1567
1628
  """
1568
- Specifies the flow(s) that this flow depends on.
1629
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1630
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1569
1631
 
1570
- ```
1571
- @trigger_on_finish(flow='FooFlow')
1572
- ```
1573
- or
1574
- ```
1575
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1576
- ```
1577
- This decorator respects the @project decorator and triggers the flow
1578
- when upstream runs within the same namespace complete successfully
1579
1632
 
1580
- Additionally, you can specify project aware upstream flow dependencies
1581
- by specifying the fully qualified project_flow_name.
1582
- ```
1583
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1584
- ```
1585
- or
1586
- ```
1587
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1588
- ```
1633
+ Parameters
1634
+ ----------
1635
+ timeout : int
1636
+ Time, in seconds before the task times out and fails. (Default: 3600)
1637
+ poke_interval : int
1638
+ Time in seconds that the job should wait in between each try. (Default: 60)
1639
+ mode : str
1640
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1641
+ exponential_backoff : bool
1642
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1643
+ pool : str
1644
+ the slot pool this task should run in,
1645
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1646
+ soft_fail : bool
1647
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1648
+ name : str
1649
+ Name of the sensor on Airflow
1650
+ description : str
1651
+ Description of sensor in the Airflow UI
1652
+ external_dag_id : str
1653
+ The dag_id that contains the task you want to wait for.
1654
+ external_task_ids : List[str]
1655
+ The list of task_ids that you want to wait for.
1656
+ If None (default value) the sensor waits for the DAG. (Default: None)
1657
+ allowed_states : List[str]
1658
+ Iterable of allowed states, (Default: ['success'])
1659
+ failed_states : List[str]
1660
+ Iterable of failed or dis-allowed states. (Default: None)
1661
+ execution_delta : datetime.timedelta
1662
+ time difference with the previous execution to look at,
1663
+ the default is the same logical date as the current task or DAG. (Default: None)
1664
+ check_existence: bool
1665
+ Set to True to check if the external task exists or check if
1666
+ the DAG to wait for exists. (Default: True)
1667
+ """
1668
+ ...
1669
+
1670
+ @typing.overload
1671
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1672
+ """
1673
+ Specifies the PyPI packages for all steps of the flow.
1589
1674
 
1590
- You can also specify just the project or project branch (other values will be
1591
- inferred from the current project or project branch):
1592
- ```
1593
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1594
- ```
1675
+ Use `@pypi_base` to set common packages required by all
1676
+ steps and use `@pypi` to specify step-specific overrides.
1595
1677
 
1596
- Note that `branch` is typically one of:
1597
- - `prod`
1598
- - `user.bob`
1599
- - `test.my_experiment`
1600
- - `prod.staging`
1678
+ Parameters
1679
+ ----------
1680
+ packages : Dict[str, str], default: {}
1681
+ Packages to use for this flow. The key is the name of the package
1682
+ and the value is the version to use.
1683
+ python : str, optional, default: None
1684
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1685
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1686
+ """
1687
+ ...
1688
+
1689
+ @typing.overload
1690
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1691
+ ...
1692
+
1693
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1694
+ """
1695
+ Specifies the PyPI packages for all steps of the flow.
1601
1696
 
1697
+ Use `@pypi_base` to set common packages required by all
1698
+ steps and use `@pypi` to specify step-specific overrides.
1602
1699
 
1603
1700
  Parameters
1604
1701
  ----------
1605
- flow : Union[str, Dict[str, str]], optional, default None
1606
- Upstream flow dependency for this flow.
1607
- flows : List[Union[str, Dict[str, str]]], default []
1608
- Upstream flow dependencies for this flow.
1609
- options : Dict[str, Any], default {}
1610
- Backend-specific configuration for tuning eventing behavior.
1702
+ packages : Dict[str, str], default: {}
1703
+ Packages to use for this flow. The key is the name of the package
1704
+ and the value is the version to use.
1705
+ python : str, optional, default: None
1706
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1707
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1611
1708
  """
1612
1709
  ...
1613
1710
 
@@ -1690,53 +1787,53 @@ def project(*, name: str, branch: typing.Optional[str] = None, production: bool
1690
1787
  ...
1691
1788
 
1692
1789
  @typing.overload
1693
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1790
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1694
1791
  """
1695
- Specifies the Conda environment for all steps of the flow.
1696
-
1697
- Use `@conda_base` to set common libraries required by all
1698
- steps and use `@conda` to specify step-specific additions.
1792
+ Specifies the times when the flow should be run when running on a
1793
+ production scheduler.
1699
1794
 
1700
1795
 
1701
1796
  Parameters
1702
1797
  ----------
1703
- packages : Dict[str, str], default {}
1704
- Packages to use for this flow. The key is the name of the package
1705
- and the value is the version to use.
1706
- libraries : Dict[str, str], default {}
1707
- Supported for backward compatibility. When used with packages, packages will take precedence.
1708
- python : str, optional, default None
1709
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1710
- that the version used will correspond to the version of the Python interpreter used to start the run.
1711
- disabled : bool, default False
1712
- If set to True, disables Conda.
1798
+ hourly : bool, default False
1799
+ Run the workflow hourly.
1800
+ daily : bool, default True
1801
+ Run the workflow daily.
1802
+ weekly : bool, default False
1803
+ Run the workflow weekly.
1804
+ cron : str, optional, default None
1805
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1806
+ specified by this expression.
1807
+ timezone : str, optional, default None
1808
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1809
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1713
1810
  """
1714
1811
  ...
1715
1812
 
1716
1813
  @typing.overload
1717
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1814
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1718
1815
  ...
1719
1816
 
1720
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1817
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1721
1818
  """
1722
- Specifies the Conda environment for all steps of the flow.
1723
-
1724
- Use `@conda_base` to set common libraries required by all
1725
- steps and use `@conda` to specify step-specific additions.
1819
+ Specifies the times when the flow should be run when running on a
1820
+ production scheduler.
1726
1821
 
1727
1822
 
1728
1823
  Parameters
1729
1824
  ----------
1730
- packages : Dict[str, str], default {}
1731
- Packages to use for this flow. The key is the name of the package
1732
- and the value is the version to use.
1733
- libraries : Dict[str, str], default {}
1734
- Supported for backward compatibility. When used with packages, packages will take precedence.
1735
- python : str, optional, default None
1736
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1737
- that the version used will correspond to the version of the Python interpreter used to start the run.
1738
- disabled : bool, default False
1739
- If set to True, disables Conda.
1825
+ hourly : bool, default False
1826
+ Run the workflow hourly.
1827
+ daily : bool, default True
1828
+ Run the workflow daily.
1829
+ weekly : bool, default False
1830
+ Run the workflow weekly.
1831
+ cron : str, optional, default None
1832
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1833
+ specified by this expression.
1834
+ timezone : str, optional, default None
1835
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1836
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1740
1837
  """
1741
1838
  ...
1742
1839
 
@@ -1766,319 +1863,222 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
1766
1863
 
1767
1864
  'parameters' can also be a list of strings and tuples like so:
1768
1865
  ```
1769
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1770
- ```
1771
- This is equivalent to:
1772
- ```
1773
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1774
- ```
1775
-
1776
-
1777
- Parameters
1778
- ----------
1779
- event : Union[str, Dict[str, Any]], optional, default None
1780
- Event dependency for this flow.
1781
- events : List[Union[str, Dict[str, Any]]], default []
1782
- Events dependency for this flow.
1783
- options : Dict[str, Any], default {}
1784
- Backend-specific configuration for tuning eventing behavior.
1785
- """
1786
- ...
1787
-
1788
- @typing.overload
1789
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1790
- ...
1791
-
1792
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1793
- """
1794
- Specifies the event(s) that this flow depends on.
1795
-
1796
- ```
1797
- @trigger(event='foo')
1798
- ```
1799
- or
1800
- ```
1801
- @trigger(events=['foo', 'bar'])
1802
- ```
1803
-
1804
- Additionally, you can specify the parameter mappings
1805
- to map event payload to Metaflow parameters for the flow.
1806
- ```
1807
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1808
- ```
1809
- or
1810
- ```
1811
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1812
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1813
- ```
1814
-
1815
- 'parameters' can also be a list of strings and tuples like so:
1816
- ```
1817
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1818
- ```
1819
- This is equivalent to:
1820
- ```
1821
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1822
- ```
1823
-
1824
-
1825
- Parameters
1826
- ----------
1827
- event : Union[str, Dict[str, Any]], optional, default None
1828
- Event dependency for this flow.
1829
- events : List[Union[str, Dict[str, Any]]], default []
1830
- Events dependency for this flow.
1831
- options : Dict[str, Any], default {}
1832
- Backend-specific configuration for tuning eventing behavior.
1833
- """
1834
- ...
1835
-
1836
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1837
- """
1838
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1839
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1840
-
1841
-
1842
- Parameters
1843
- ----------
1844
- timeout : int
1845
- Time, in seconds before the task times out and fails. (Default: 3600)
1846
- poke_interval : int
1847
- Time in seconds that the job should wait in between each try. (Default: 60)
1848
- mode : str
1849
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1850
- exponential_backoff : bool
1851
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1852
- pool : str
1853
- the slot pool this task should run in,
1854
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1855
- soft_fail : bool
1856
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1857
- name : str
1858
- Name of the sensor on Airflow
1859
- description : str
1860
- Description of sensor in the Airflow UI
1861
- external_dag_id : str
1862
- The dag_id that contains the task you want to wait for.
1863
- external_task_ids : List[str]
1864
- The list of task_ids that you want to wait for.
1865
- If None (default value) the sensor waits for the DAG. (Default: None)
1866
- allowed_states : List[str]
1867
- Iterable of allowed states, (Default: ['success'])
1868
- failed_states : List[str]
1869
- Iterable of failed or dis-allowed states. (Default: None)
1870
- execution_delta : datetime.timedelta
1871
- time difference with the previous execution to look at,
1872
- the default is the same logical date as the current task or DAG. (Default: None)
1873
- check_existence: bool
1874
- Set to True to check if the external task exists or check if
1875
- the DAG to wait for exists. (Default: True)
1876
- """
1877
- ...
1878
-
1879
- def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1880
- """
1881
- Allows setting external datastores to save data for the
1882
- `@checkpoint`/`@model`/`@huggingface_hub` decorators.
1883
-
1884
- This decorator is useful when users wish to save data to a different datastore
1885
- than what is configured in Metaflow. This can be for variety of reasons:
1886
-
1887
- 1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
1888
- 2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
1889
- - Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
1890
- 3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
1891
- - Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
1892
-
1893
- Usage:
1894
- ----------
1895
-
1896
- - Using a custom IAM role to access the datastore.
1897
-
1898
- ```python
1899
- @with_artifact_store(
1900
- type="s3",
1901
- config=lambda: {
1902
- "root": "s3://my-bucket-foo/path/to/root",
1903
- "role_arn": ROLE,
1904
- },
1905
- )
1906
- class MyFlow(FlowSpec):
1907
-
1908
- @checkpoint
1909
- @step
1910
- def start(self):
1911
- with open("my_file.txt", "w") as f:
1912
- f.write("Hello, World!")
1913
- self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1914
- self.next(self.end)
1915
-
1916
- ```
1917
-
1918
- - Using credentials to access the s3-compatible datastore.
1919
-
1920
- ```python
1921
- @with_artifact_store(
1922
- type="s3",
1923
- config=lambda: {
1924
- "root": "s3://my-bucket-foo/path/to/root",
1925
- "client_params": {
1926
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1927
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1928
- },
1929
- },
1930
- )
1931
- class MyFlow(FlowSpec):
1866
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1867
+ ```
1868
+ This is equivalent to:
1869
+ ```
1870
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1871
+ ```
1932
1872
 
1933
- @checkpoint
1934
- @step
1935
- def start(self):
1936
- with open("my_file.txt", "w") as f:
1937
- f.write("Hello, World!")
1938
- self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
1939
- self.next(self.end)
1940
1873
 
1941
- ```
1874
+ Parameters
1875
+ ----------
1876
+ event : Union[str, Dict[str, Any]], optional, default None
1877
+ Event dependency for this flow.
1878
+ events : List[Union[str, Dict[str, Any]]], default []
1879
+ Events dependency for this flow.
1880
+ options : Dict[str, Any], default {}
1881
+ Backend-specific configuration for tuning eventing behavior.
1882
+ """
1883
+ ...
1884
+
1885
+ @typing.overload
1886
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1887
+ ...
1888
+
1889
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1890
+ """
1891
+ Specifies the event(s) that this flow depends on.
1942
1892
 
1943
- - Accessing objects stored in external datastores after task execution.
1893
+ ```
1894
+ @trigger(event='foo')
1895
+ ```
1896
+ or
1897
+ ```
1898
+ @trigger(events=['foo', 'bar'])
1899
+ ```
1944
1900
 
1945
- ```python
1946
- run = Run("CheckpointsTestsFlow/8992")
1947
- with artifact_store_from(run=run, config={
1948
- "client_params": {
1949
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1950
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1951
- },
1952
- }):
1953
- with Checkpoint() as cp:
1954
- latest = cp.list(
1955
- task=run["start"].task
1956
- )[0]
1957
- print(latest)
1958
- cp.load(
1959
- latest,
1960
- "test-checkpoints"
1961
- )
1901
+ Additionally, you can specify the parameter mappings
1902
+ to map event payload to Metaflow parameters for the flow.
1903
+ ```
1904
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1905
+ ```
1906
+ or
1907
+ ```
1908
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1909
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1910
+ ```
1962
1911
 
1963
- task = Task("TorchTuneFlow/8484/train/53673")
1964
- with artifact_store_from(run=run, config={
1965
- "client_params": {
1966
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1967
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1968
- },
1969
- }):
1970
- load_model(
1971
- task.data.model_ref,
1972
- "test-models"
1973
- )
1974
- ```
1975
- Parameters:
1976
- ----------
1912
+ 'parameters' can also be a list of strings and tuples like so:
1913
+ ```
1914
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1915
+ ```
1916
+ This is equivalent to:
1917
+ ```
1918
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1919
+ ```
1977
1920
 
1978
- type: str
1979
- The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1980
1921
 
1981
- config: dict or Callable
1982
- Dictionary of configuration options for the datastore. The following keys are required:
1983
- - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1984
- - example: 's3://bucket-name/path/to/root'
1985
- - example: 'gs://bucket-name/path/to/root'
1986
- - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1987
- - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1988
- - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1989
- - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1922
+ Parameters
1923
+ ----------
1924
+ event : Union[str, Dict[str, Any]], optional, default None
1925
+ Event dependency for this flow.
1926
+ events : List[Union[str, Dict[str, Any]]], default []
1927
+ Events dependency for this flow.
1928
+ options : Dict[str, Any], default {}
1929
+ Backend-specific configuration for tuning eventing behavior.
1990
1930
  """
1991
1931
  ...
1992
1932
 
1993
1933
  @typing.overload
1994
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1934
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1995
1935
  """
1996
- Specifies the PyPI packages for all steps of the flow.
1936
+ Specifies the Conda environment for all steps of the flow.
1937
+
1938
+ Use `@conda_base` to set common libraries required by all
1939
+ steps and use `@conda` to specify step-specific additions.
1997
1940
 
1998
- Use `@pypi_base` to set common packages required by all
1999
- steps and use `@pypi` to specify step-specific overrides.
2000
1941
 
2001
1942
  Parameters
2002
1943
  ----------
2003
- packages : Dict[str, str], default: {}
1944
+ packages : Dict[str, str], default {}
2004
1945
  Packages to use for this flow. The key is the name of the package
2005
1946
  and the value is the version to use.
2006
- python : str, optional, default: None
1947
+ libraries : Dict[str, str], default {}
1948
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1949
+ python : str, optional, default None
2007
1950
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
2008
1951
  that the version used will correspond to the version of the Python interpreter used to start the run.
1952
+ disabled : bool, default False
1953
+ If set to True, disables Conda.
2009
1954
  """
2010
1955
  ...
2011
1956
 
2012
1957
  @typing.overload
2013
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1958
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
2014
1959
  ...
2015
1960
 
2016
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1961
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
2017
1962
  """
2018
- Specifies the PyPI packages for all steps of the flow.
1963
+ Specifies the Conda environment for all steps of the flow.
1964
+
1965
+ Use `@conda_base` to set common libraries required by all
1966
+ steps and use `@conda` to specify step-specific additions.
2019
1967
 
2020
- Use `@pypi_base` to set common packages required by all
2021
- steps and use `@pypi` to specify step-specific overrides.
2022
1968
 
2023
1969
  Parameters
2024
1970
  ----------
2025
- packages : Dict[str, str], default: {}
1971
+ packages : Dict[str, str], default {}
2026
1972
  Packages to use for this flow. The key is the name of the package
2027
1973
  and the value is the version to use.
2028
- python : str, optional, default: None
1974
+ libraries : Dict[str, str], default {}
1975
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1976
+ python : str, optional, default None
2029
1977
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
2030
1978
  that the version used will correspond to the version of the Python interpreter used to start the run.
1979
+ disabled : bool, default False
1980
+ If set to True, disables Conda.
2031
1981
  """
2032
1982
  ...
2033
1983
 
2034
1984
  @typing.overload
2035
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1985
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
2036
1986
  """
2037
- Specifies the times when the flow should be run when running on a
2038
- production scheduler.
1987
+ Specifies the flow(s) that this flow depends on.
1988
+
1989
+ ```
1990
+ @trigger_on_finish(flow='FooFlow')
1991
+ ```
1992
+ or
1993
+ ```
1994
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1995
+ ```
1996
+ This decorator respects the @project decorator and triggers the flow
1997
+ when upstream runs within the same namespace complete successfully
1998
+
1999
+ Additionally, you can specify project aware upstream flow dependencies
2000
+ by specifying the fully qualified project_flow_name.
2001
+ ```
2002
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
2003
+ ```
2004
+ or
2005
+ ```
2006
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
2007
+ ```
2008
+
2009
+ You can also specify just the project or project branch (other values will be
2010
+ inferred from the current project or project branch):
2011
+ ```
2012
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
2013
+ ```
2014
+
2015
+ Note that `branch` is typically one of:
2016
+ - `prod`
2017
+ - `user.bob`
2018
+ - `test.my_experiment`
2019
+ - `prod.staging`
2039
2020
 
2040
2021
 
2041
2022
  Parameters
2042
2023
  ----------
2043
- hourly : bool, default False
2044
- Run the workflow hourly.
2045
- daily : bool, default True
2046
- Run the workflow daily.
2047
- weekly : bool, default False
2048
- Run the workflow weekly.
2049
- cron : str, optional, default None
2050
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
2051
- specified by this expression.
2052
- timezone : str, optional, default None
2053
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
2054
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
2024
+ flow : Union[str, Dict[str, str]], optional, default None
2025
+ Upstream flow dependency for this flow.
2026
+ flows : List[Union[str, Dict[str, str]]], default []
2027
+ Upstream flow dependencies for this flow.
2028
+ options : Dict[str, Any], default {}
2029
+ Backend-specific configuration for tuning eventing behavior.
2055
2030
  """
2056
2031
  ...
2057
2032
 
2058
2033
  @typing.overload
2059
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
2034
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
2060
2035
  ...
2061
2036
 
2062
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
2037
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
2063
2038
  """
2064
- Specifies the times when the flow should be run when running on a
2065
- production scheduler.
2039
+ Specifies the flow(s) that this flow depends on.
2040
+
2041
+ ```
2042
+ @trigger_on_finish(flow='FooFlow')
2043
+ ```
2044
+ or
2045
+ ```
2046
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
2047
+ ```
2048
+ This decorator respects the @project decorator and triggers the flow
2049
+ when upstream runs within the same namespace complete successfully
2050
+
2051
+ Additionally, you can specify project aware upstream flow dependencies
2052
+ by specifying the fully qualified project_flow_name.
2053
+ ```
2054
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
2055
+ ```
2056
+ or
2057
+ ```
2058
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
2059
+ ```
2060
+
2061
+ You can also specify just the project or project branch (other values will be
2062
+ inferred from the current project or project branch):
2063
+ ```
2064
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
2065
+ ```
2066
+
2067
+ Note that `branch` is typically one of:
2068
+ - `prod`
2069
+ - `user.bob`
2070
+ - `test.my_experiment`
2071
+ - `prod.staging`
2066
2072
 
2067
2073
 
2068
2074
  Parameters
2069
2075
  ----------
2070
- hourly : bool, default False
2071
- Run the workflow hourly.
2072
- daily : bool, default True
2073
- Run the workflow daily.
2074
- weekly : bool, default False
2075
- Run the workflow weekly.
2076
- cron : str, optional, default None
2077
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
2078
- specified by this expression.
2079
- timezone : str, optional, default None
2080
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
2081
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
2076
+ flow : Union[str, Dict[str, str]], optional, default None
2077
+ Upstream flow dependency for this flow.
2078
+ flows : List[Union[str, Dict[str, str]]], default []
2079
+ Upstream flow dependencies for this flow.
2080
+ options : Dict[str, Any], default {}
2081
+ Backend-specific configuration for tuning eventing behavior.
2082
2082
  """
2083
2083
  ...
2084
2084