ob-metaflow-stubs 6.0.10.11__py2.py3-none-any.whl → 6.0.10.13__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (266) hide show
  1. metaflow-stubs/__init__.pyi +1006 -1005
  2. metaflow-stubs/cards.pyi +5 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +7 -7
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +6 -6
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +12 -5
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +52 -52
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +5 -5
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +4 -4
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/__init__.pyi +6 -0
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/cards/hf_hub_card.pyi +93 -0
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +6 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  64. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  65. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  66. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +3 -3
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +5 -5
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +4 -4
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +6 -6
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +3 -3
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +3 -3
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +3 -3
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +3 -3
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +4 -4
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  110. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  111. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  112. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  113. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  114. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +3 -3
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  116. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  117. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
  118. metaflow-stubs/multicore_utils.pyi +2 -2
  119. metaflow-stubs/ob_internal.pyi +2 -2
  120. metaflow-stubs/packaging_sys/__init__.pyi +8 -8
  121. metaflow-stubs/packaging_sys/backend.pyi +2 -2
  122. metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
  123. metaflow-stubs/packaging_sys/tar_backend.pyi +5 -5
  124. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  125. metaflow-stubs/packaging_sys/v1.pyi +4 -4
  126. metaflow-stubs/parameters.pyi +3 -3
  127. metaflow-stubs/plugins/__init__.pyi +14 -13
  128. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  129. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  130. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  131. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  132. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  133. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  134. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  135. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  136. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  137. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  138. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  139. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  140. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  141. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  142. metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
  143. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  144. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  145. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  146. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  147. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  148. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  149. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  150. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  152. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  153. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  154. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  155. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  157. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  158. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  159. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  160. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  161. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  162. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  163. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  164. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  165. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  166. metaflow-stubs/plugins/cards/__init__.pyi +6 -6
  167. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  168. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  169. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  170. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  171. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +3 -2
  172. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  173. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  174. metaflow-stubs/plugins/cards/card_modules/components.pyi +168 -4
  175. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  176. metaflow-stubs/plugins/cards/card_modules/json_viewer.pyi +119 -0
  177. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  178. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  179. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  180. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  181. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  182. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  183. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  184. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  185. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  186. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  187. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  188. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  189. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  190. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  191. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  192. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  193. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  194. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  195. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  196. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  197. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  198. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  199. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  200. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  201. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  202. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  203. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  204. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  205. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  206. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  207. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  208. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  209. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  210. metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
  211. metaflow-stubs/plugins/optuna/__init__.pyi +2 -2
  212. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  213. metaflow-stubs/plugins/parsers.pyi +25 -0
  214. metaflow-stubs/plugins/perimeters.pyi +2 -2
  215. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  216. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  217. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  218. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  219. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  220. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  221. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  222. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  223. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  224. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  225. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  226. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  227. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  228. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  229. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  230. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  231. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  232. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  233. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  234. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  235. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  236. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  237. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  238. metaflow-stubs/profilers/__init__.pyi +2 -2
  239. metaflow-stubs/pylint_wrapper.pyi +2 -2
  240. metaflow-stubs/runner/__init__.pyi +2 -2
  241. metaflow-stubs/runner/deployer.pyi +5 -5
  242. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  243. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  244. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  245. metaflow-stubs/runner/nbrun.pyi +2 -2
  246. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  247. metaflow-stubs/runner/utils.pyi +3 -3
  248. metaflow-stubs/system/__init__.pyi +2 -2
  249. metaflow-stubs/system/system_logger.pyi +3 -3
  250. metaflow-stubs/system/system_monitor.pyi +2 -2
  251. metaflow-stubs/tagging_util.pyi +2 -2
  252. metaflow-stubs/tuple_util.pyi +2 -2
  253. metaflow-stubs/user_configs/__init__.pyi +2 -2
  254. metaflow-stubs/user_configs/config_options.pyi +3 -3
  255. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  256. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  257. metaflow-stubs/user_decorators/common.pyi +2 -2
  258. metaflow-stubs/user_decorators/mutable_flow.pyi +4 -4
  259. metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
  260. metaflow-stubs/user_decorators/user_flow_decorator.pyi +5 -5
  261. metaflow-stubs/user_decorators/user_step_decorator.pyi +5 -5
  262. {ob_metaflow_stubs-6.0.10.11.dist-info → ob_metaflow_stubs-6.0.10.13.dist-info}/METADATA +1 -1
  263. ob_metaflow_stubs-6.0.10.13.dist-info/RECORD +266 -0
  264. ob_metaflow_stubs-6.0.10.11.dist-info/RECORD +0 -262
  265. {ob_metaflow_stubs-6.0.10.11.dist-info → ob_metaflow_stubs-6.0.10.13.dist-info}/WHEEL +0 -0
  266. {ob_metaflow_stubs-6.0.10.11.dist-info → ob_metaflow_stubs-6.0.10.13.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.18.7.5+obcheckpoint(0.2.7);ob(v1) #
4
- # Generated on 2025-09-29T21:43:14.782191 #
3
+ # MF version: 2.18.10.1+obcheckpoint(0.2.8);ob(v1) #
4
+ # Generated on 2025-10-02T16:55:23.853861 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -39,18 +39,19 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
39
39
  from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
+ from . import tuple_util as tuple_util
42
43
  from . import cards as cards
43
44
  from . import metaflow_git as metaflow_git
44
- from . import tuple_util as tuple_util
45
45
  from . import events as events
46
46
  from . import runner as runner
47
47
  from . import plugins as plugins
48
48
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
49
49
  from . import includefile as includefile
50
50
  from .includefile import IncludeFile as IncludeFile
51
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
52
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
51
+ from .plugins.parsers import yaml_parser as yaml_parser
53
52
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
53
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
54
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
54
55
  from . import client as client
55
56
  from .client.core import namespace as namespace
56
57
  from .client.core import get_namespace as get_namespace
@@ -224,308 +225,286 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
224
225
  ...
225
226
 
226
227
  @typing.overload
227
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
228
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
228
229
  """
229
- A simple decorator that demonstrates using CardDecoratorInjector
230
- to inject a card and render simple markdown content.
230
+ Enables loading / saving of models within a step.
231
+
232
+ > Examples
233
+ - Saving Models
234
+ ```python
235
+ @model
236
+ @step
237
+ def train(self):
238
+ # current.model.save returns a dictionary reference to the model saved
239
+ self.my_model = current.model.save(
240
+ path_to_my_model,
241
+ label="my_model",
242
+ metadata={
243
+ "epochs": 10,
244
+ "batch-size": 32,
245
+ "learning-rate": 0.001,
246
+ }
247
+ )
248
+ self.next(self.test)
249
+
250
+ @model(load="my_model")
251
+ @step
252
+ def test(self):
253
+ # `current.model.loaded` returns a dictionary of the loaded models
254
+ # where the key is the name of the artifact and the value is the path to the model
255
+ print(os.listdir(current.model.loaded["my_model"]))
256
+ self.next(self.end)
257
+ ```
258
+
259
+ - Loading models
260
+ ```python
261
+ @step
262
+ def train(self):
263
+ # current.model.load returns the path to the model loaded
264
+ checkpoint_path = current.model.load(
265
+ self.checkpoint_key,
266
+ )
267
+ model_path = current.model.load(
268
+ self.model,
269
+ )
270
+ self.next(self.test)
271
+ ```
272
+
273
+
274
+ Parameters
275
+ ----------
276
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
277
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
278
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
279
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
280
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
281
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
282
+
283
+ temp_dir_root : str, default: None
284
+ The root directory under which `current.model.loaded` will store loaded models
231
285
  """
232
286
  ...
233
287
 
234
288
  @typing.overload
235
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
289
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
236
290
  ...
237
291
 
238
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
239
- """
240
- A simple decorator that demonstrates using CardDecoratorInjector
241
- to inject a card and render simple markdown content.
242
- """
292
+ @typing.overload
293
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
243
294
  ...
244
295
 
245
- @typing.overload
246
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
296
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
247
297
  """
248
- Creates a human-readable report, a Metaflow Card, after this step completes.
298
+ Enables loading / saving of models within a step.
249
299
 
250
- Note that you may add multiple `@card` decorators in a step with different parameters.
300
+ > Examples
301
+ - Saving Models
302
+ ```python
303
+ @model
304
+ @step
305
+ def train(self):
306
+ # current.model.save returns a dictionary reference to the model saved
307
+ self.my_model = current.model.save(
308
+ path_to_my_model,
309
+ label="my_model",
310
+ metadata={
311
+ "epochs": 10,
312
+ "batch-size": 32,
313
+ "learning-rate": 0.001,
314
+ }
315
+ )
316
+ self.next(self.test)
317
+
318
+ @model(load="my_model")
319
+ @step
320
+ def test(self):
321
+ # `current.model.loaded` returns a dictionary of the loaded models
322
+ # where the key is the name of the artifact and the value is the path to the model
323
+ print(os.listdir(current.model.loaded["my_model"]))
324
+ self.next(self.end)
325
+ ```
326
+
327
+ - Loading models
328
+ ```python
329
+ @step
330
+ def train(self):
331
+ # current.model.load returns the path to the model loaded
332
+ checkpoint_path = current.model.load(
333
+ self.checkpoint_key,
334
+ )
335
+ model_path = current.model.load(
336
+ self.model,
337
+ )
338
+ self.next(self.test)
339
+ ```
251
340
 
252
341
 
253
342
  Parameters
254
343
  ----------
255
- type : str, default 'default'
256
- Card type.
257
- id : str, optional, default None
258
- If multiple cards are present, use this id to identify this card.
259
- options : Dict[str, Any], default {}
260
- Options passed to the card. The contents depend on the card type.
261
- timeout : int, default 45
262
- Interrupt reporting if it takes more than this many seconds.
344
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
345
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
346
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
347
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
348
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
349
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
350
+
351
+ temp_dir_root : str, default: None
352
+ The root directory under which `current.model.loaded` will store loaded models
263
353
  """
264
354
  ...
265
355
 
266
356
  @typing.overload
267
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
357
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
358
+ """
359
+ Decorator prototype for all step decorators. This function gets specialized
360
+ and imported for all decorators types by _import_plugin_decorators().
361
+ """
268
362
  ...
269
363
 
270
364
  @typing.overload
271
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
365
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
272
366
  ...
273
367
 
274
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
368
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
275
369
  """
276
- Creates a human-readable report, a Metaflow Card, after this step completes.
277
-
278
- Note that you may add multiple `@card` decorators in a step with different parameters.
370
+ Decorator prototype for all step decorators. This function gets specialized
371
+ and imported for all decorators types by _import_plugin_decorators().
372
+ """
373
+ ...
374
+
375
+ @typing.overload
376
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
377
+ """
378
+ Specifies environment variables to be set prior to the execution of a step.
279
379
 
280
380
 
281
381
  Parameters
282
382
  ----------
283
- type : str, default 'default'
284
- Card type.
285
- id : str, optional, default None
286
- If multiple cards are present, use this id to identify this card.
287
- options : Dict[str, Any], default {}
288
- Options passed to the card. The contents depend on the card type.
289
- timeout : int, default 45
290
- Interrupt reporting if it takes more than this many seconds.
383
+ vars : Dict[str, str], default {}
384
+ Dictionary of environment variables to set.
291
385
  """
292
386
  ...
293
387
 
294
- def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
388
+ @typing.overload
389
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
390
+ ...
391
+
392
+ @typing.overload
393
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
394
+ ...
395
+
396
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
295
397
  """
296
- This decorator is used to run vllm APIs as Metaflow task sidecars.
297
-
298
- User code call
299
- --------------
300
- @vllm(
301
- model="...",
302
- ...
303
- )
304
-
305
- Valid backend options
306
- ---------------------
307
- - 'local': Run as a separate process on the local task machine.
308
-
309
- Valid model options
310
- -------------------
311
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
312
-
313
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
314
- If you need multiple models, you must create multiple @vllm decorators.
398
+ Specifies environment variables to be set prior to the execution of a step.
315
399
 
316
400
 
317
401
  Parameters
318
402
  ----------
319
- model: str
320
- HuggingFace model identifier to be served by vLLM.
321
- backend: str
322
- Determines where and how to run the vLLM process.
323
- openai_api_server: bool
324
- Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
325
- Default is False (uses native engine).
326
- Set to True for backward compatibility with existing code.
327
- debug: bool
328
- Whether to turn on verbose debugging logs.
329
- card_refresh_interval: int
330
- Interval in seconds for refreshing the vLLM status card.
331
- Only used when openai_api_server=True.
332
- max_retries: int
333
- Maximum number of retries checking for vLLM server startup.
334
- Only used when openai_api_server=True.
335
- retry_alert_frequency: int
336
- Frequency of alert logs for vLLM server startup retries.
337
- Only used when openai_api_server=True.
338
- engine_args : dict
339
- Additional keyword arguments to pass to the vLLM engine.
340
- For example, `tensor_parallel_size=2`.
403
+ vars : Dict[str, str], default {}
404
+ Dictionary of environment variables to set.
341
405
  """
342
406
  ...
343
407
 
344
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
408
+ @typing.overload
409
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
345
410
  """
346
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
347
-
348
- User code call
349
- --------------
350
- @ollama(
351
- models=[...],
352
- ...
353
- )
354
-
355
- Valid backend options
356
- ---------------------
357
- - 'local': Run as a separate process on the local task machine.
358
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
359
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
411
+ Specifies the Conda environment for the step.
360
412
 
361
- Valid model options
362
- -------------------
363
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
413
+ Information in this decorator will augment any
414
+ attributes set in the `@conda_base` flow-level decorator. Hence,
415
+ you can use `@conda_base` to set packages required by all
416
+ steps and use `@conda` to specify step-specific overrides.
364
417
 
365
418
 
366
419
  Parameters
367
420
  ----------
368
- models: list[str]
369
- List of Ollama containers running models in sidecars.
370
- backend: str
371
- Determines where and how to run the Ollama process.
372
- force_pull: bool
373
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
374
- cache_update_policy: str
375
- Cache update policy: "auto", "force", or "never".
376
- force_cache_update: bool
377
- Simple override for "force" cache update policy.
378
- debug: bool
379
- Whether to turn on verbose debugging logs.
380
- circuit_breaker_config: dict
381
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
382
- timeout_config: dict
383
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
384
- """
385
- ...
386
-
387
- @typing.overload
388
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
389
- """
390
- Specifies the PyPI packages for the step.
391
-
392
- Information in this decorator will augment any
393
- attributes set in the `@pyi_base` flow-level decorator. Hence,
394
- you can use `@pypi_base` to set packages required by all
395
- steps and use `@pypi` to specify step-specific overrides.
396
-
397
-
398
- Parameters
399
- ----------
400
- packages : Dict[str, str], default: {}
421
+ packages : Dict[str, str], default {}
401
422
  Packages to use for this step. The key is the name of the package
402
423
  and the value is the version to use.
403
- python : str, optional, default: None
424
+ libraries : Dict[str, str], default {}
425
+ Supported for backward compatibility. When used with packages, packages will take precedence.
426
+ python : str, optional, default None
404
427
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
405
428
  that the version used will correspond to the version of the Python interpreter used to start the run.
429
+ disabled : bool, default False
430
+ If set to True, disables @conda.
406
431
  """
407
432
  ...
408
433
 
409
434
  @typing.overload
410
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
435
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
411
436
  ...
412
437
 
413
438
  @typing.overload
414
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
439
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
415
440
  ...
416
441
 
417
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
442
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
418
443
  """
419
- Specifies the PyPI packages for the step.
444
+ Specifies the Conda environment for the step.
420
445
 
421
446
  Information in this decorator will augment any
422
- attributes set in the `@pyi_base` flow-level decorator. Hence,
423
- you can use `@pypi_base` to set packages required by all
424
- steps and use `@pypi` to specify step-specific overrides.
447
+ attributes set in the `@conda_base` flow-level decorator. Hence,
448
+ you can use `@conda_base` to set packages required by all
449
+ steps and use `@conda` to specify step-specific overrides.
425
450
 
426
451
 
427
452
  Parameters
428
453
  ----------
429
- packages : Dict[str, str], default: {}
454
+ packages : Dict[str, str], default {}
430
455
  Packages to use for this step. The key is the name of the package
431
456
  and the value is the version to use.
432
- python : str, optional, default: None
457
+ libraries : Dict[str, str], default {}
458
+ Supported for backward compatibility. When used with packages, packages will take precedence.
459
+ python : str, optional, default None
433
460
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
434
461
  that the version used will correspond to the version of the Python interpreter used to start the run.
462
+ disabled : bool, default False
463
+ If set to True, disables @conda.
435
464
  """
436
465
  ...
437
466
 
438
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
467
+ @typing.overload
468
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
439
469
  """
440
- Specifies that this step should execute on Kubernetes.
470
+ Specifies secrets to be retrieved and injected as environment variables prior to
471
+ the execution of a step.
441
472
 
442
473
 
443
474
  Parameters
444
475
  ----------
445
- cpu : int, default 1
446
- Number of CPUs required for this step. If `@resources` is
447
- also present, the maximum value from all decorators is used.
448
- memory : int, default 4096
449
- Memory size (in MB) required for this step. If
450
- `@resources` is also present, the maximum value from all decorators is
451
- used.
452
- disk : int, default 10240
453
- Disk size (in MB) required for this step. If
454
- `@resources` is also present, the maximum value from all decorators is
455
- used.
456
- image : str, optional, default None
457
- Docker image to use when launching on Kubernetes. If not specified, and
458
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
459
- not, a default Docker image mapping to the current version of Python is used.
460
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
461
- If given, the imagePullPolicy to be applied to the Docker image of the step.
462
- image_pull_secrets: List[str], default []
463
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
464
- Kubernetes image pull secrets to use when pulling container images
465
- in Kubernetes.
466
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
467
- Kubernetes service account to use when launching pod in Kubernetes.
468
- secrets : List[str], optional, default None
469
- Kubernetes secrets to use when launching pod in Kubernetes. These
470
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
471
- in Metaflow configuration.
472
- node_selector: Union[Dict[str,str], str], optional, default None
473
- Kubernetes node selector(s) to apply to the pod running the task.
474
- Can be passed in as a comma separated string of values e.g.
475
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
476
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
477
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
478
- Kubernetes namespace to use when launching pod in Kubernetes.
479
- gpu : int, optional, default None
480
- Number of GPUs required for this step. A value of zero implies that
481
- the scheduled node should not have GPUs.
482
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
483
- The vendor of the GPUs to be used for this step.
484
- tolerations : List[Dict[str,str]], default []
485
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
486
- Kubernetes tolerations to use when launching pod in Kubernetes.
487
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
488
- Kubernetes labels to use when launching pod in Kubernetes.
489
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
490
- Kubernetes annotations to use when launching pod in Kubernetes.
491
- use_tmpfs : bool, default False
492
- This enables an explicit tmpfs mount for this step.
493
- tmpfs_tempdir : bool, default True
494
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
495
- tmpfs_size : int, optional, default: None
496
- The value for the size (in MiB) of the tmpfs mount for this step.
497
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
498
- memory allocated for this step.
499
- tmpfs_path : str, optional, default /metaflow_temp
500
- Path to tmpfs mount for this step.
501
- persistent_volume_claims : Dict[str, str], optional, default None
502
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
503
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
504
- shared_memory: int, optional
505
- Shared memory size (in MiB) required for this step
506
- port: int, optional
507
- Port number to specify in the Kubernetes job object
508
- compute_pool : str, optional, default None
509
- Compute pool to be used for for this step.
510
- If not specified, any accessible compute pool within the perimeter is used.
511
- hostname_resolution_timeout: int, default 10 * 60
512
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
513
- Only applicable when @parallel is used.
514
- qos: str, default: Burstable
515
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
476
+ sources : List[Union[str, Dict[str, Any]]], default: []
477
+ List of secret specs, defining how the secrets are to be retrieved
478
+ role : str, optional, default: None
479
+ Role to use for fetching secrets
480
+ """
481
+ ...
482
+
483
+ @typing.overload
484
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
485
+ ...
486
+
487
+ @typing.overload
488
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
489
+ ...
490
+
491
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
492
+ """
493
+ Specifies secrets to be retrieved and injected as environment variables prior to
494
+ the execution of a step.
516
495
 
517
- security_context: Dict[str, Any], optional, default None
518
- Container security context. Applies to the task container. Allows the following keys:
519
- - privileged: bool, optional, default None
520
- - allow_privilege_escalation: bool, optional, default None
521
- - run_as_user: int, optional, default None
522
- - run_as_group: int, optional, default None
523
- - run_as_non_root: bool, optional, default None
496
+
497
+ Parameters
498
+ ----------
499
+ sources : List[Union[str, Dict[str, Any]]], default: []
500
+ List of secret specs, defining how the secrets are to be retrieved
501
+ role : str, optional, default: None
502
+ Role to use for fetching secrets
524
503
  """
525
504
  ...
526
505
 
527
506
  @typing.overload
528
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
507
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
529
508
  """
530
509
  Decorator prototype for all step decorators. This function gets specialized
531
510
  and imported for all decorators types by _import_plugin_decorators().
@@ -533,401 +512,256 @@ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.
533
512
  ...
534
513
 
535
514
  @typing.overload
536
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
515
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
537
516
  ...
538
517
 
539
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
518
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
540
519
  """
541
520
  Decorator prototype for all step decorators. This function gets specialized
542
521
  and imported for all decorators types by _import_plugin_decorators().
543
522
  """
544
523
  ...
545
524
 
546
- @typing.overload
547
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
525
+ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
548
526
  """
549
- Enables checkpointing for a step.
550
-
551
- > Examples
552
-
553
- - Saving Checkpoints
527
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
554
528
 
555
- ```python
556
- @checkpoint
557
- @step
558
- def train(self):
559
- model = create_model(self.parameters, checkpoint_path = None)
560
- for i in range(self.epochs):
561
- # some training logic
562
- loss = model.train(self.dataset)
563
- if i % 10 == 0:
564
- model.save(
565
- current.checkpoint.directory,
566
- )
567
- # saves the contents of the `current.checkpoint.directory` as a checkpoint
568
- # and returns a reference dictionary to the checkpoint saved in the datastore
569
- self.latest_checkpoint = current.checkpoint.save(
570
- name="epoch_checkpoint",
571
- metadata={
572
- "epoch": i,
573
- "loss": loss,
574
- }
575
- )
576
- ```
529
+ User code call
530
+ --------------
531
+ @vllm(
532
+ model="...",
533
+ ...
534
+ )
577
535
 
578
- - Using Loaded Checkpoints
536
+ Valid backend options
537
+ ---------------------
538
+ - 'local': Run as a separate process on the local task machine.
579
539
 
580
- ```python
581
- @retry(times=3)
582
- @checkpoint
583
- @step
584
- def train(self):
585
- # Assume that the task has restarted and the previous attempt of the task
586
- # saved a checkpoint
587
- checkpoint_path = None
588
- if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
589
- print("Loaded checkpoint from the previous attempt")
590
- checkpoint_path = current.checkpoint.directory
540
+ Valid model options
541
+ -------------------
542
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
591
543
 
592
- model = create_model(self.parameters, checkpoint_path = checkpoint_path)
593
- for i in range(self.epochs):
594
- ...
595
- ```
544
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
545
+ If you need multiple models, you must create multiple @vllm decorators.
596
546
 
597
547
 
598
548
  Parameters
599
549
  ----------
600
- load_policy : str, default: "fresh"
601
- The policy for loading the checkpoint. The following policies are supported:
602
- - "eager": Loads the the latest available checkpoint within the namespace.
603
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
604
- will be loaded at the start of the task.
605
- - "none": Do not load any checkpoint
606
- - "fresh": Loads the lastest checkpoint created within the running Task.
607
- This mode helps loading checkpoints across various retry attempts of the same task.
608
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
609
- created within the task will be loaded when the task is retries execution on failure.
610
-
611
- temp_dir_root : str, default: None
612
- The root directory under which `current.checkpoint.directory` will be created.
613
- """
614
- ...
615
-
616
- @typing.overload
617
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
618
- ...
619
-
620
- @typing.overload
621
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
622
- ...
550
+ model: str
551
+ HuggingFace model identifier to be served by vLLM.
552
+ backend: str
553
+ Determines where and how to run the vLLM process.
554
+ openai_api_server: bool
555
+ Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
556
+ Default is False (uses native engine).
557
+ Set to True for backward compatibility with existing code.
558
+ debug: bool
559
+ Whether to turn on verbose debugging logs.
560
+ card_refresh_interval: int
561
+ Interval in seconds for refreshing the vLLM status card.
562
+ Only used when openai_api_server=True.
563
+ max_retries: int
564
+ Maximum number of retries checking for vLLM server startup.
565
+ Only used when openai_api_server=True.
566
+ retry_alert_frequency: int
567
+ Frequency of alert logs for vLLM server startup retries.
568
+ Only used when openai_api_server=True.
569
+ engine_args : dict
570
+ Additional keyword arguments to pass to the vLLM engine.
571
+ For example, `tensor_parallel_size=2`.
572
+ """
573
+ ...
623
574
 
624
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
575
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
625
576
  """
626
- Enables checkpointing for a step.
627
-
628
- > Examples
629
-
630
- - Saving Checkpoints
631
-
632
- ```python
633
- @checkpoint
634
- @step
635
- def train(self):
636
- model = create_model(self.parameters, checkpoint_path = None)
637
- for i in range(self.epochs):
638
- # some training logic
639
- loss = model.train(self.dataset)
640
- if i % 10 == 0:
641
- model.save(
642
- current.checkpoint.directory,
643
- )
644
- # saves the contents of the `current.checkpoint.directory` as a checkpoint
645
- # and returns a reference dictionary to the checkpoint saved in the datastore
646
- self.latest_checkpoint = current.checkpoint.save(
647
- name="epoch_checkpoint",
648
- metadata={
649
- "epoch": i,
650
- "loss": loss,
651
- }
652
- )
653
- ```
577
+ Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
578
+ for S3 read and write requests.
654
579
 
655
- - Using Loaded Checkpoints
580
+ This decorator requires an integration in the Outerbounds platform that
581
+ points to an external bucket. It affects S3 operations performed via
582
+ Metaflow's `get_aws_client` and `S3` within a `@step`.
656
583
 
657
- ```python
658
- @retry(times=3)
659
- @checkpoint
660
- @step
661
- def train(self):
662
- # Assume that the task has restarted and the previous attempt of the task
663
- # saved a checkpoint
664
- checkpoint_path = None
665
- if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
666
- print("Loaded checkpoint from the previous attempt")
667
- checkpoint_path = current.checkpoint.directory
584
+ Read operations
585
+ ---------------
586
+ All read operations pass through the proxy. If an object does not already
587
+ exist in the external bucket, it is cached there. For example, if code reads
588
+ from buckets `FOO` and `BAR` using the `S3` interface, objects from both
589
+ buckets are cached in the external bucket.
668
590
 
669
- model = create_model(self.parameters, checkpoint_path = checkpoint_path)
670
- for i in range(self.epochs):
671
- ...
672
- ```
591
+ During task execution, all S3‑related read requests are routed through the
592
+ proxy:
593
+ - If the object is present in the external object store, the proxy
594
+ streams it directly from there without accessing the requested origin
595
+ bucket.
596
+ - If the object is not present in the external storage, the proxy
597
+ fetches it from the requested bucket, caches it in the external
598
+ storage, and streams the response from the origin bucket.
673
599
 
600
+ Warning
601
+ -------
602
+ All READ operations (e.g., GetObject, HeadObject) pass through the external
603
+ bucket regardless of the bucket specified in user code. Even
604
+ `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
605
+ external bucket cache.
674
606
 
675
- Parameters
676
- ----------
677
- load_policy : str, default: "fresh"
678
- The policy for loading the checkpoint. The following policies are supported:
679
- - "eager": Loads the the latest available checkpoint within the namespace.
680
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
681
- will be loaded at the start of the task.
682
- - "none": Do not load any checkpoint
683
- - "fresh": Loads the lastest checkpoint created within the running Task.
684
- This mode helps loading checkpoints across various retry attempts of the same task.
685
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
686
- created within the task will be loaded when the task is retries execution on failure.
607
+ Write operations
608
+ ----------------
609
+ Write behavior is controlled by the `write_mode` parameter, which determines
610
+ whether writes also persist objects in the cache.
687
611
 
688
- temp_dir_root : str, default: None
689
- The root directory under which `current.checkpoint.directory` will be created.
690
- """
691
- ...
692
-
693
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
694
- """
695
- Specifies that this step should execute on DGX cloud.
612
+ `write_mode` values:
613
+ - `origin-and-cache`: objects are written both to the cache and to their
614
+ intended origin bucket.
615
+ - `origin`: objects are written only to their intended origin bucket.
696
616
 
697
617
 
698
618
  Parameters
699
619
  ----------
700
- gpu : int
701
- Number of GPUs to use.
702
- gpu_type : str
703
- Type of Nvidia GPU to use.
620
+ integration_name : str, optional
621
+ [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
622
+ that holds the configuration for the external, S3‑compatible object
623
+ storage bucket. If not specified, the only available S3 proxy
624
+ integration in the namespace is used (fails if multiple exist).
625
+ write_mode : str, optional
626
+ Controls whether writes also go to the external bucket.
627
+ - `origin` (default)
628
+ - `origin-and-cache`
629
+ debug : bool, optional
630
+ Enables debug logging for proxy operations.
704
631
  """
705
632
  ...
706
633
 
707
634
  @typing.overload
708
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
635
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
709
636
  """
710
- Specifies that the step will success under all circumstances.
711
-
712
- The decorator will create an optional artifact, specified by `var`, which
713
- contains the exception raised. You can use it to detect the presence
714
- of errors, indicating that all happy-path artifacts produced by the step
715
- are missing.
716
-
717
-
718
- Parameters
719
- ----------
720
- var : str, optional, default None
721
- Name of the artifact in which to store the caught exception.
722
- If not specified, the exception is not stored.
723
- print_exception : bool, default True
724
- Determines whether or not the exception is printed to
725
- stdout when caught.
637
+ Internal decorator to support Fast bakery
726
638
  """
727
639
  ...
728
640
 
729
641
  @typing.overload
730
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
731
- ...
732
-
733
- @typing.overload
734
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
642
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
735
643
  ...
736
644
 
737
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
645
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
738
646
  """
739
- Specifies that the step will success under all circumstances.
740
-
741
- The decorator will create an optional artifact, specified by `var`, which
742
- contains the exception raised. You can use it to detect the presence
743
- of errors, indicating that all happy-path artifacts produced by the step
744
- are missing.
745
-
746
-
747
- Parameters
748
- ----------
749
- var : str, optional, default None
750
- Name of the artifact in which to store the caught exception.
751
- If not specified, the exception is not stored.
752
- print_exception : bool, default True
753
- Determines whether or not the exception is printed to
754
- stdout when caught.
647
+ Internal decorator to support Fast bakery
755
648
  """
756
649
  ...
757
650
 
758
- @typing.overload
759
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
651
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
760
652
  """
761
- Specifies the Conda environment for the step.
653
+ Decorator that helps cache, version, and store models/datasets from the Hugging Face Hub.
762
654
 
763
- Information in this decorator will augment any
764
- attributes set in the `@conda_base` flow-level decorator. Hence,
765
- you can use `@conda_base` to set packages required by all
766
- steps and use `@conda` to specify step-specific overrides.
655
+ Examples
656
+ --------
767
657
 
658
+ ```python
659
+ # **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
660
+ @huggingface_hub
661
+ @step
662
+ def pull_model_from_huggingface(self):
663
+ # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
664
+ # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
665
+ # with the same `repo_id` in the backend storage, it will not download the model again. The return
666
+ # value of the function is a reference to the model in the backend storage.
667
+ # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
768
668
 
769
- Parameters
770
- ----------
771
- packages : Dict[str, str], default {}
772
- Packages to use for this step. The key is the name of the package
773
- and the value is the version to use.
774
- libraries : Dict[str, str], default {}
775
- Supported for backward compatibility. When used with packages, packages will take precedence.
776
- python : str, optional, default None
777
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
778
- that the version used will correspond to the version of the Python interpreter used to start the run.
779
- disabled : bool, default False
780
- If set to True, disables @conda.
781
- """
782
- ...
783
-
784
- @typing.overload
785
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
786
- ...
787
-
788
- @typing.overload
789
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
790
- ...
791
-
792
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
793
- """
794
- Specifies the Conda environment for the step.
669
+ self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
670
+ self.llama_model = current.huggingface_hub.snapshot_download(
671
+ repo_id=self.model_id,
672
+ allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
673
+ )
674
+ self.next(self.train)
795
675
 
796
- Information in this decorator will augment any
797
- attributes set in the `@conda_base` flow-level decorator. Hence,
798
- you can use `@conda_base` to set packages required by all
799
- steps and use `@conda` to specify step-specific overrides.
676
+ # **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
677
+ @huggingface_hub
678
+ @step
679
+ def run_training(self):
680
+ # Temporary directory (auto-cleaned on exit)
681
+ with current.huggingface_hub.load(
682
+ repo_id="google-bert/bert-base-uncased",
683
+ allow_patterns=["*.bin"],
684
+ ) as local_path:
685
+ # Use files under local_path
686
+ train_model(local_path)
687
+ ...
800
688
 
689
+ # **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
801
690
 
802
- Parameters
803
- ----------
804
- packages : Dict[str, str], default {}
805
- Packages to use for this step. The key is the name of the package
806
- and the value is the version to use.
807
- libraries : Dict[str, str], default {}
808
- Supported for backward compatibility. When used with packages, packages will take precedence.
809
- python : str, optional, default None
810
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
811
- that the version used will correspond to the version of the Python interpreter used to start the run.
812
- disabled : bool, default False
813
- If set to True, disables @conda.
814
- """
815
- ...
816
-
817
- @typing.overload
818
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
819
- """
820
- Specifies secrets to be retrieved and injected as environment variables prior to
821
- the execution of a step.
691
+ @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
692
+ @step
693
+ def pull_model_from_huggingface(self):
694
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
822
695
 
696
+ @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
697
+ @step
698
+ def finetune_model(self):
699
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
700
+ # path_to_model will be /my-directory
823
701
 
824
- Parameters
825
- ----------
826
- sources : List[Union[str, Dict[str, Any]]], default: []
827
- List of secret specs, defining how the secrets are to be retrieved
828
- role : str, optional, default: None
829
- Role to use for fetching secrets
830
- """
831
- ...
832
-
833
- @typing.overload
834
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
835
- ...
836
-
837
- @typing.overload
838
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
839
- ...
840
-
841
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
842
- """
843
- Specifies secrets to be retrieved and injected as environment variables prior to
844
- the execution of a step.
702
+
703
+ # Takes all the arguments passed to `snapshot_download`
704
+ # except for `local_dir`
705
+ @huggingface_hub(load=[
706
+ {
707
+ "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
708
+ },
709
+ {
710
+ "repo_id": "myorg/mistral-lora",
711
+ "repo_type": "model",
712
+ },
713
+ ])
714
+ @step
715
+ def finetune_model(self):
716
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
717
+ # path_to_model will be /my-directory
718
+ ```
845
719
 
846
720
 
847
721
  Parameters
848
722
  ----------
849
- sources : List[Union[str, Dict[str, Any]]], default: []
850
- List of secret specs, defining how the secrets are to be retrieved
851
- role : str, optional, default: None
852
- Role to use for fetching secrets
853
- """
854
- ...
855
-
856
- @typing.overload
857
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
858
- """
859
- Decorator prototype for all step decorators. This function gets specialized
860
- and imported for all decorators types by _import_plugin_decorators().
861
- """
862
- ...
863
-
864
- @typing.overload
865
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
866
- ...
867
-
868
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
869
- """
870
- Decorator prototype for all step decorators. This function gets specialized
871
- and imported for all decorators types by _import_plugin_decorators().
872
- """
873
- ...
874
-
875
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
876
- """
877
- Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
878
- for S3 read and write requests.
879
-
880
- This decorator requires an integration in the Outerbounds platform that
881
- points to an external bucket. It affects S3 operations performed via
882
- Metaflow's `get_aws_client` and `S3` within a `@step`.
723
+ temp_dir_root : str, optional
724
+ The root directory that will hold the temporary directory where objects will be downloaded.
883
725
 
884
- Read operations
885
- ---------------
886
- All read operations pass through the proxy. If an object does not already
887
- exist in the external bucket, it is cached there. For example, if code reads
888
- from buckets `FOO` and `BAR` using the `S3` interface, objects from both
889
- buckets are cached in the external bucket.
726
+ cache_scope : str, optional
727
+ The scope of the cache. Can be `checkpoint` / `flow` / `global`.
728
+ - `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
729
+ i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
730
+ Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
890
731
 
891
- During task execution, all S3‑related read requests are routed through the
892
- proxy:
893
- - If the object is present in the external object store, the proxy
894
- streams it directly from there without accessing the requested origin
895
- bucket.
896
- - If the object is not present in the external storage, the proxy
897
- fetches it from the requested bucket, caches it in the external
898
- storage, and streams the response from the origin bucket.
732
+ - `flow`: All repos are cached under the flow, regardless of namespace.
733
+ i.e., the cached path is derived solely from the flow name.
734
+ When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
899
735
 
900
- Warning
901
- -------
902
- All READ operations (e.g., GetObject, HeadObject) pass through the external
903
- bucket regardless of the bucket specified in user code. Even
904
- `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
905
- external bucket cache.
736
+ - `global`: All repos are cached under a globally static path.
737
+ i.e., the base path of the cache is static and all repos are stored under it.
738
+ When to use this mode:
739
+ - All repos from the Hugging Face Hub need to be shared by users across all flow executions.
740
+ - Each caching scope comes with its own trade-offs:
741
+ - `checkpoint`:
742
+ - Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
743
+ - Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
744
+ - `flow`:
745
+ - Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
746
+ - The blast radius of a bad checkpoint is limited to all runs of a particular flow.
747
+ - It doesn't promote cache reuse across flows.
748
+ - `global`:
749
+ - Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
750
+ - It promotes cache reuse across flows.
751
+ - The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
906
752
 
907
- Write operations
908
- ----------------
909
- Write behavior is controlled by the `write_mode` parameter, which determines
910
- whether writes also persist objects in the cache.
753
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
754
+ The list of repos (models/datasets) to load.
911
755
 
912
- `write_mode` values:
913
- - `origin-and-cache`: objects are written both to the cache and to their
914
- intended origin bucket.
915
- - `origin`: objects are written only to their intended origin bucket.
756
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
916
757
 
758
+ - If repo (model/dataset) is not found in the datastore:
759
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
760
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
761
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
917
762
 
918
- Parameters
919
- ----------
920
- integration_name : str, optional
921
- [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
922
- that holds the configuration for the external, S3‑compatible object
923
- storage bucket. If not specified, the only available S3 proxy
924
- integration in the namespace is used (fails if multiple exist).
925
- write_mode : str, optional
926
- Controls whether writes also go to the external bucket.
927
- - `origin` (default)
928
- - `origin-and-cache`
929
- debug : bool, optional
930
- Enables debug logging for proxy operations.
763
+ - If repo is found in the datastore:
764
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
931
765
  """
932
766
  ...
933
767
 
@@ -1010,165 +844,111 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
1010
844
  """
1011
845
  ...
1012
846
 
1013
- @typing.overload
1014
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
847
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1015
848
  """
1016
- Enables loading / saving of models within a step.
1017
-
1018
- > Examples
1019
- - Saving Models
1020
- ```python
1021
- @model
1022
- @step
1023
- def train(self):
1024
- # current.model.save returns a dictionary reference to the model saved
1025
- self.my_model = current.model.save(
1026
- path_to_my_model,
1027
- label="my_model",
1028
- metadata={
1029
- "epochs": 10,
1030
- "batch-size": 32,
1031
- "learning-rate": 0.001,
1032
- }
1033
- )
1034
- self.next(self.test)
1035
-
1036
- @model(load="my_model")
1037
- @step
1038
- def test(self):
1039
- # `current.model.loaded` returns a dictionary of the loaded models
1040
- # where the key is the name of the artifact and the value is the path to the model
1041
- print(os.listdir(current.model.loaded["my_model"]))
1042
- self.next(self.end)
1043
- ```
1044
-
1045
- - Loading models
1046
- ```python
1047
- @step
1048
- def train(self):
1049
- # current.model.load returns the path to the model loaded
1050
- checkpoint_path = current.model.load(
1051
- self.checkpoint_key,
1052
- )
1053
- model_path = current.model.load(
1054
- self.model,
1055
- )
1056
- self.next(self.test)
1057
- ```
849
+ Specifies that this step should execute on Kubernetes.
1058
850
 
1059
851
 
1060
852
  Parameters
1061
853
  ----------
1062
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1063
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1064
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1065
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1066
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1067
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1068
-
1069
- temp_dir_root : str, default: None
1070
- The root directory under which `current.model.loaded` will store loaded models
1071
- """
1072
- ...
1073
-
1074
- @typing.overload
1075
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1076
- ...
1077
-
1078
- @typing.overload
1079
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1080
- ...
1081
-
1082
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1083
- """
1084
- Enables loading / saving of models within a step.
1085
-
1086
- > Examples
1087
- - Saving Models
1088
- ```python
1089
- @model
1090
- @step
1091
- def train(self):
1092
- # current.model.save returns a dictionary reference to the model saved
1093
- self.my_model = current.model.save(
1094
- path_to_my_model,
1095
- label="my_model",
1096
- metadata={
1097
- "epochs": 10,
1098
- "batch-size": 32,
1099
- "learning-rate": 0.001,
1100
- }
1101
- )
1102
- self.next(self.test)
1103
-
1104
- @model(load="my_model")
1105
- @step
1106
- def test(self):
1107
- # `current.model.loaded` returns a dictionary of the loaded models
1108
- # where the key is the name of the artifact and the value is the path to the model
1109
- print(os.listdir(current.model.loaded["my_model"]))
1110
- self.next(self.end)
1111
- ```
1112
-
1113
- - Loading models
1114
- ```python
1115
- @step
1116
- def train(self):
1117
- # current.model.load returns the path to the model loaded
1118
- checkpoint_path = current.model.load(
1119
- self.checkpoint_key,
1120
- )
1121
- model_path = current.model.load(
1122
- self.model,
1123
- )
1124
- self.next(self.test)
1125
- ```
1126
-
1127
-
1128
- Parameters
1129
- ----------
1130
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1131
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1132
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1133
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1134
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1135
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
854
+ cpu : int, default 1
855
+ Number of CPUs required for this step. If `@resources` is
856
+ also present, the maximum value from all decorators is used.
857
+ memory : int, default 4096
858
+ Memory size (in MB) required for this step. If
859
+ `@resources` is also present, the maximum value from all decorators is
860
+ used.
861
+ disk : int, default 10240
862
+ Disk size (in MB) required for this step. If
863
+ `@resources` is also present, the maximum value from all decorators is
864
+ used.
865
+ image : str, optional, default None
866
+ Docker image to use when launching on Kubernetes. If not specified, and
867
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
868
+ not, a default Docker image mapping to the current version of Python is used.
869
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
870
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
871
+ image_pull_secrets: List[str], default []
872
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
873
+ Kubernetes image pull secrets to use when pulling container images
874
+ in Kubernetes.
875
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
876
+ Kubernetes service account to use when launching pod in Kubernetes.
877
+ secrets : List[str], optional, default None
878
+ Kubernetes secrets to use when launching pod in Kubernetes. These
879
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
880
+ in Metaflow configuration.
881
+ node_selector: Union[Dict[str,str], str], optional, default None
882
+ Kubernetes node selector(s) to apply to the pod running the task.
883
+ Can be passed in as a comma separated string of values e.g.
884
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
885
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
886
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
887
+ Kubernetes namespace to use when launching pod in Kubernetes.
888
+ gpu : int, optional, default None
889
+ Number of GPUs required for this step. A value of zero implies that
890
+ the scheduled node should not have GPUs.
891
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
892
+ The vendor of the GPUs to be used for this step.
893
+ tolerations : List[Dict[str,str]], default []
894
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
895
+ Kubernetes tolerations to use when launching pod in Kubernetes.
896
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
897
+ Kubernetes labels to use when launching pod in Kubernetes.
898
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
899
+ Kubernetes annotations to use when launching pod in Kubernetes.
900
+ use_tmpfs : bool, default False
901
+ This enables an explicit tmpfs mount for this step.
902
+ tmpfs_tempdir : bool, default True
903
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
904
+ tmpfs_size : int, optional, default: None
905
+ The value for the size (in MiB) of the tmpfs mount for this step.
906
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
907
+ memory allocated for this step.
908
+ tmpfs_path : str, optional, default /metaflow_temp
909
+ Path to tmpfs mount for this step.
910
+ persistent_volume_claims : Dict[str, str], optional, default None
911
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
912
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
913
+ shared_memory: int, optional
914
+ Shared memory size (in MiB) required for this step
915
+ port: int, optional
916
+ Port number to specify in the Kubernetes job object
917
+ compute_pool : str, optional, default None
918
+ Compute pool to be used for for this step.
919
+ If not specified, any accessible compute pool within the perimeter is used.
920
+ hostname_resolution_timeout: int, default 10 * 60
921
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
922
+ Only applicable when @parallel is used.
923
+ qos: str, default: Burstable
924
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
1136
925
 
1137
- temp_dir_root : str, default: None
1138
- The root directory under which `current.model.loaded` will store loaded models
926
+ security_context: Dict[str, Any], optional, default None
927
+ Container security context. Applies to the task container. Allows the following keys:
928
+ - privileged: bool, optional, default None
929
+ - allow_privilege_escalation: bool, optional, default None
930
+ - run_as_user: int, optional, default None
931
+ - run_as_group: int, optional, default None
932
+ - run_as_non_root: bool, optional, default None
1139
933
  """
1140
934
  ...
1141
935
 
1142
936
  @typing.overload
1143
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
937
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1144
938
  """
1145
- Specifies environment variables to be set prior to the execution of a step.
1146
-
1147
-
1148
- Parameters
1149
- ----------
1150
- vars : Dict[str, str], default {}
1151
- Dictionary of environment variables to set.
939
+ A simple decorator that demonstrates using CardDecoratorInjector
940
+ to inject a card and render simple markdown content.
1152
941
  """
1153
942
  ...
1154
943
 
1155
944
  @typing.overload
1156
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1157
- ...
1158
-
1159
- @typing.overload
1160
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
945
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1161
946
  ...
1162
947
 
1163
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
948
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1164
949
  """
1165
- Specifies environment variables to be set prior to the execution of a step.
1166
-
1167
-
1168
- Parameters
1169
- ----------
1170
- vars : Dict[str, str], default {}
1171
- Dictionary of environment variables to set.
950
+ A simple decorator that demonstrates using CardDecoratorInjector
951
+ to inject a card and render simple markdown content.
1172
952
  """
1173
953
  ...
1174
954
 
@@ -1295,120 +1075,167 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
1295
1075
  """
1296
1076
  ...
1297
1077
 
1298
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1078
+ @typing.overload
1079
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1299
1080
  """
1300
- Decorator that helps cache, version, and store models/datasets from the Hugging Face Hub.
1301
-
1302
- Examples
1303
- --------
1081
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1304
1082
 
1305
- ```python
1306
- # **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
1307
- @huggingface_hub
1308
- @step
1309
- def pull_model_from_huggingface(self):
1310
- # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
1311
- # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
1312
- # with the same `repo_id` in the backend storage, it will not download the model again. The return
1313
- # value of the function is a reference to the model in the backend storage.
1314
- # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
1083
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1315
1084
 
1316
- self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
1317
- self.llama_model = current.huggingface_hub.snapshot_download(
1318
- repo_id=self.model_id,
1319
- allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
1320
- )
1321
- self.next(self.train)
1322
1085
 
1323
- # **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
1324
- @huggingface_hub
1325
- @step
1326
- def run_training(self):
1327
- # Temporary directory (auto-cleaned on exit)
1328
- with current.huggingface_hub.load(
1329
- repo_id="google-bert/bert-base-uncased",
1330
- allow_patterns=["*.bin"],
1331
- ) as local_path:
1332
- # Use files under local_path
1333
- train_model(local_path)
1334
- ...
1086
+ Parameters
1087
+ ----------
1088
+ type : str, default 'default'
1089
+ Card type.
1090
+ id : str, optional, default None
1091
+ If multiple cards are present, use this id to identify this card.
1092
+ options : Dict[str, Any], default {}
1093
+ Options passed to the card. The contents depend on the card type.
1094
+ timeout : int, default 45
1095
+ Interrupt reporting if it takes more than this many seconds.
1096
+ """
1097
+ ...
1098
+
1099
+ @typing.overload
1100
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1101
+ ...
1102
+
1103
+ @typing.overload
1104
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1105
+ ...
1106
+
1107
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1108
+ """
1109
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1335
1110
 
1336
- # **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
1111
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1337
1112
 
1338
- @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
1339
- @step
1340
- def pull_model_from_huggingface(self):
1341
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
1342
1113
 
1343
- @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
1344
- @step
1345
- def finetune_model(self):
1346
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
1347
- # path_to_model will be /my-directory
1114
+ Parameters
1115
+ ----------
1116
+ type : str, default 'default'
1117
+ Card type.
1118
+ id : str, optional, default None
1119
+ If multiple cards are present, use this id to identify this card.
1120
+ options : Dict[str, Any], default {}
1121
+ Options passed to the card. The contents depend on the card type.
1122
+ timeout : int, default 45
1123
+ Interrupt reporting if it takes more than this many seconds.
1124
+ """
1125
+ ...
1126
+
1127
+ def nebius_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1128
+ """
1129
+ `@nebius_s3_proxy` is a Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1130
+ It exists to make it easier for users to know that this decorator should only be used with
1131
+ a Neo Cloud like Nebius. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
1348
1132
 
1349
1133
 
1350
- # Takes all the arguments passed to `snapshot_download`
1351
- # except for `local_dir`
1352
- @huggingface_hub(load=[
1353
- {
1354
- "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
1355
- },
1356
- {
1357
- "repo_id": "myorg/mistral-lora",
1358
- "repo_type": "model",
1359
- },
1360
- ])
1361
- @step
1362
- def finetune_model(self):
1363
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
1364
- # path_to_model will be /my-directory
1365
- ```
1134
+ Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
1135
+ for S3 read and write requests.
1136
+
1137
+ This decorator requires an integration in the Outerbounds platform that
1138
+ points to an external bucket. It affects S3 operations performed via
1139
+ Metaflow's `get_aws_client` and `S3` within a `@step`.
1140
+
1141
+ Read operations
1142
+ ---------------
1143
+ All read operations pass through the proxy. If an object does not already
1144
+ exist in the external bucket, it is cached there. For example, if code reads
1145
+ from buckets `FOO` and `BAR` using the `S3` interface, objects from both
1146
+ buckets are cached in the external bucket.
1147
+
1148
+ During task execution, all S3‑related read requests are routed through the
1149
+ proxy:
1150
+ - If the object is present in the external object store, the proxy
1151
+ streams it directly from there without accessing the requested origin
1152
+ bucket.
1153
+ - If the object is not present in the external storage, the proxy
1154
+ fetches it from the requested bucket, caches it in the external
1155
+ storage, and streams the response from the origin bucket.
1156
+
1157
+ Warning
1158
+ -------
1159
+ All READ operations (e.g., GetObject, HeadObject) pass through the external
1160
+ bucket regardless of the bucket specified in user code. Even
1161
+ `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
1162
+ external bucket cache.
1163
+
1164
+ Write operations
1165
+ ----------------
1166
+ Write behavior is controlled by the `write_mode` parameter, which determines
1167
+ whether writes also persist objects in the cache.
1168
+
1169
+ `write_mode` values:
1170
+ - `origin-and-cache`: objects are written both to the cache and to their
1171
+ intended origin bucket.
1172
+ - `origin`: objects are written only to their intended origin bucket.
1366
1173
 
1367
1174
 
1368
1175
  Parameters
1369
1176
  ----------
1370
- temp_dir_root : str, optional
1371
- The root directory that will hold the temporary directory where objects will be downloaded.
1372
-
1373
- cache_scope : str, optional
1374
- The scope of the cache. Can be `checkpoint` / `flow` / `global`.
1375
- - `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
1376
- i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
1377
- Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
1177
+ integration_name : str, optional
1178
+ [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
1179
+ that holds the configuration for the external, S3‑compatible object
1180
+ storage bucket. If not specified, the only available S3 proxy
1181
+ integration in the namespace is used (fails if multiple exist).
1182
+ write_mode : str, optional
1183
+ Controls whether writes also go to the external bucket.
1184
+ - `origin` (default)
1185
+ - `origin-and-cache`
1186
+ debug : bool, optional
1187
+ Enables debug logging for proxy operations.
1188
+ """
1189
+ ...
1190
+
1191
+ @typing.overload
1192
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1193
+ """
1194
+ Specifies that the step will success under all circumstances.
1378
1195
 
1379
- - `flow`: All repos are cached under the flow, regardless of namespace.
1380
- i.e., the cached path is derived solely from the flow name.
1381
- When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
1196
+ The decorator will create an optional artifact, specified by `var`, which
1197
+ contains the exception raised. You can use it to detect the presence
1198
+ of errors, indicating that all happy-path artifacts produced by the step
1199
+ are missing.
1382
1200
 
1383
- - `global`: All repos are cached under a globally static path.
1384
- i.e., the base path of the cache is static and all repos are stored under it.
1385
- When to use this mode:
1386
- - All repos from the Hugging Face Hub need to be shared by users across all flow executions.
1387
- - Each caching scope comes with its own trade-offs:
1388
- - `checkpoint`:
1389
- - Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
1390
- - Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
1391
- - `flow`:
1392
- - Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
1393
- - The blast radius of a bad checkpoint is limited to all runs of a particular flow.
1394
- - It doesn't promote cache reuse across flows.
1395
- - `global`:
1396
- - Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
1397
- - It promotes cache reuse across flows.
1398
- - The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
1399
1201
 
1400
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
1401
- The list of repos (models/datasets) to load.
1202
+ Parameters
1203
+ ----------
1204
+ var : str, optional, default None
1205
+ Name of the artifact in which to store the caught exception.
1206
+ If not specified, the exception is not stored.
1207
+ print_exception : bool, default True
1208
+ Determines whether or not the exception is printed to
1209
+ stdout when caught.
1210
+ """
1211
+ ...
1212
+
1213
+ @typing.overload
1214
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1215
+ ...
1216
+
1217
+ @typing.overload
1218
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1219
+ ...
1220
+
1221
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1222
+ """
1223
+ Specifies that the step will success under all circumstances.
1402
1224
 
1403
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
1225
+ The decorator will create an optional artifact, specified by `var`, which
1226
+ contains the exception raised. You can use it to detect the presence
1227
+ of errors, indicating that all happy-path artifacts produced by the step
1228
+ are missing.
1404
1229
 
1405
- - If repo (model/dataset) is not found in the datastore:
1406
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
1407
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
1408
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
1409
1230
 
1410
- - If repo is found in the datastore:
1411
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
1231
+ Parameters
1232
+ ----------
1233
+ var : str, optional, default None
1234
+ Name of the artifact in which to store the caught exception.
1235
+ If not specified, the exception is not stored.
1236
+ print_exception : bool, default True
1237
+ Determines whether or not the exception is printed to
1238
+ stdout when caught.
1412
1239
  """
1413
1240
  ...
1414
1241
 
@@ -1428,84 +1255,336 @@ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[t
1428
1255
  """
1429
1256
  ...
1430
1257
 
1258
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1259
+ """
1260
+ Specifies that this step should execute on DGX cloud.
1261
+
1262
+
1263
+ Parameters
1264
+ ----------
1265
+ gpu : int
1266
+ Number of GPUs to use.
1267
+ gpu_type : str
1268
+ Type of Nvidia GPU to use.
1269
+ """
1270
+ ...
1271
+
1272
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1273
+ """
1274
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
1275
+
1276
+ User code call
1277
+ --------------
1278
+ @ollama(
1279
+ models=[...],
1280
+ ...
1281
+ )
1282
+
1283
+ Valid backend options
1284
+ ---------------------
1285
+ - 'local': Run as a separate process on the local task machine.
1286
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
1287
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
1288
+
1289
+ Valid model options
1290
+ -------------------
1291
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
1292
+
1293
+
1294
+ Parameters
1295
+ ----------
1296
+ models: list[str]
1297
+ List of Ollama containers running models in sidecars.
1298
+ backend: str
1299
+ Determines where and how to run the Ollama process.
1300
+ force_pull: bool
1301
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
1302
+ cache_update_policy: str
1303
+ Cache update policy: "auto", "force", or "never".
1304
+ force_cache_update: bool
1305
+ Simple override for "force" cache update policy.
1306
+ debug: bool
1307
+ Whether to turn on verbose debugging logs.
1308
+ circuit_breaker_config: dict
1309
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
1310
+ timeout_config: dict
1311
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
1312
+ """
1313
+ ...
1314
+
1431
1315
  @typing.overload
1432
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1316
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1433
1317
  """
1434
- Internal decorator to support Fast bakery
1318
+ Specifies the PyPI packages for the step.
1319
+
1320
+ Information in this decorator will augment any
1321
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1322
+ you can use `@pypi_base` to set packages required by all
1323
+ steps and use `@pypi` to specify step-specific overrides.
1324
+
1325
+
1326
+ Parameters
1327
+ ----------
1328
+ packages : Dict[str, str], default: {}
1329
+ Packages to use for this step. The key is the name of the package
1330
+ and the value is the version to use.
1331
+ python : str, optional, default: None
1332
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1333
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1435
1334
  """
1436
1335
  ...
1437
1336
 
1438
1337
  @typing.overload
1439
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1338
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1440
1339
  ...
1441
1340
 
1442
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1341
+ @typing.overload
1342
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1343
+ ...
1344
+
1345
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1443
1346
  """
1444
- Internal decorator to support Fast bakery
1347
+ Specifies the PyPI packages for the step.
1348
+
1349
+ Information in this decorator will augment any
1350
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1351
+ you can use `@pypi_base` to set packages required by all
1352
+ steps and use `@pypi` to specify step-specific overrides.
1353
+
1354
+
1355
+ Parameters
1356
+ ----------
1357
+ packages : Dict[str, str], default: {}
1358
+ Packages to use for this step. The key is the name of the package
1359
+ and the value is the version to use.
1360
+ python : str, optional, default: None
1361
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1362
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1445
1363
  """
1446
1364
  ...
1447
1365
 
1448
- def nebius_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1366
+ @typing.overload
1367
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1449
1368
  """
1450
- `@nebius_s3_proxy` is a Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1451
- It exists to make it easier for users to know that this decorator should only be used with
1452
- a Neo Cloud like Nebius. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
1369
+ Enables checkpointing for a step.
1370
+
1371
+ > Examples
1453
1372
 
1373
+ - Saving Checkpoints
1454
1374
 
1455
- Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
1456
- for S3 read and write requests.
1375
+ ```python
1376
+ @checkpoint
1377
+ @step
1378
+ def train(self):
1379
+ model = create_model(self.parameters, checkpoint_path = None)
1380
+ for i in range(self.epochs):
1381
+ # some training logic
1382
+ loss = model.train(self.dataset)
1383
+ if i % 10 == 0:
1384
+ model.save(
1385
+ current.checkpoint.directory,
1386
+ )
1387
+ # saves the contents of the `current.checkpoint.directory` as a checkpoint
1388
+ # and returns a reference dictionary to the checkpoint saved in the datastore
1389
+ self.latest_checkpoint = current.checkpoint.save(
1390
+ name="epoch_checkpoint",
1391
+ metadata={
1392
+ "epoch": i,
1393
+ "loss": loss,
1394
+ }
1395
+ )
1396
+ ```
1457
1397
 
1458
- This decorator requires an integration in the Outerbounds platform that
1459
- points to an external bucket. It affects S3 operations performed via
1460
- Metaflow's `get_aws_client` and `S3` within a `@step`.
1398
+ - Using Loaded Checkpoints
1461
1399
 
1462
- Read operations
1463
- ---------------
1464
- All read operations pass through the proxy. If an object does not already
1465
- exist in the external bucket, it is cached there. For example, if code reads
1466
- from buckets `FOO` and `BAR` using the `S3` interface, objects from both
1467
- buckets are cached in the external bucket.
1400
+ ```python
1401
+ @retry(times=3)
1402
+ @checkpoint
1403
+ @step
1404
+ def train(self):
1405
+ # Assume that the task has restarted and the previous attempt of the task
1406
+ # saved a checkpoint
1407
+ checkpoint_path = None
1408
+ if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
1409
+ print("Loaded checkpoint from the previous attempt")
1410
+ checkpoint_path = current.checkpoint.directory
1468
1411
 
1469
- During task execution, all S3‑related read requests are routed through the
1470
- proxy:
1471
- - If the object is present in the external object store, the proxy
1472
- streams it directly from there without accessing the requested origin
1473
- bucket.
1474
- - If the object is not present in the external storage, the proxy
1475
- fetches it from the requested bucket, caches it in the external
1476
- storage, and streams the response from the origin bucket.
1412
+ model = create_model(self.parameters, checkpoint_path = checkpoint_path)
1413
+ for i in range(self.epochs):
1414
+ ...
1415
+ ```
1477
1416
 
1478
- Warning
1479
- -------
1480
- All READ operations (e.g., GetObject, HeadObject) pass through the external
1481
- bucket regardless of the bucket specified in user code. Even
1482
- `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
1483
- external bucket cache.
1484
1417
 
1485
- Write operations
1486
- ----------------
1487
- Write behavior is controlled by the `write_mode` parameter, which determines
1488
- whether writes also persist objects in the cache.
1418
+ Parameters
1419
+ ----------
1420
+ load_policy : str, default: "fresh"
1421
+ The policy for loading the checkpoint. The following policies are supported:
1422
+ - "eager": Loads the the latest available checkpoint within the namespace.
1423
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
1424
+ will be loaded at the start of the task.
1425
+ - "none": Do not load any checkpoint
1426
+ - "fresh": Loads the lastest checkpoint created within the running Task.
1427
+ This mode helps loading checkpoints across various retry attempts of the same task.
1428
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
1429
+ created within the task will be loaded when the task is retries execution on failure.
1489
1430
 
1490
- `write_mode` values:
1491
- - `origin-and-cache`: objects are written both to the cache and to their
1492
- intended origin bucket.
1493
- - `origin`: objects are written only to their intended origin bucket.
1431
+ temp_dir_root : str, default: None
1432
+ The root directory under which `current.checkpoint.directory` will be created.
1433
+ """
1434
+ ...
1435
+
1436
+ @typing.overload
1437
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1438
+ ...
1439
+
1440
+ @typing.overload
1441
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1442
+ ...
1443
+
1444
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
1445
+ """
1446
+ Enables checkpointing for a step.
1447
+
1448
+ > Examples
1449
+
1450
+ - Saving Checkpoints
1451
+
1452
+ ```python
1453
+ @checkpoint
1454
+ @step
1455
+ def train(self):
1456
+ model = create_model(self.parameters, checkpoint_path = None)
1457
+ for i in range(self.epochs):
1458
+ # some training logic
1459
+ loss = model.train(self.dataset)
1460
+ if i % 10 == 0:
1461
+ model.save(
1462
+ current.checkpoint.directory,
1463
+ )
1464
+ # saves the contents of the `current.checkpoint.directory` as a checkpoint
1465
+ # and returns a reference dictionary to the checkpoint saved in the datastore
1466
+ self.latest_checkpoint = current.checkpoint.save(
1467
+ name="epoch_checkpoint",
1468
+ metadata={
1469
+ "epoch": i,
1470
+ "loss": loss,
1471
+ }
1472
+ )
1473
+ ```
1474
+
1475
+ - Using Loaded Checkpoints
1476
+
1477
+ ```python
1478
+ @retry(times=3)
1479
+ @checkpoint
1480
+ @step
1481
+ def train(self):
1482
+ # Assume that the task has restarted and the previous attempt of the task
1483
+ # saved a checkpoint
1484
+ checkpoint_path = None
1485
+ if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
1486
+ print("Loaded checkpoint from the previous attempt")
1487
+ checkpoint_path = current.checkpoint.directory
1488
+
1489
+ model = create_model(self.parameters, checkpoint_path = checkpoint_path)
1490
+ for i in range(self.epochs):
1491
+ ...
1492
+ ```
1494
1493
 
1495
1494
 
1496
1495
  Parameters
1497
1496
  ----------
1498
- integration_name : str, optional
1499
- [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
1500
- that holds the configuration for the external, S3‑compatible object
1501
- storage bucket. If not specified, the only available S3 proxy
1502
- integration in the namespace is used (fails if multiple exist).
1503
- write_mode : str, optional
1504
- Controls whether writes also go to the external bucket.
1505
- - `origin` (default)
1506
- - `origin-and-cache`
1507
- debug : bool, optional
1508
- Enables debug logging for proxy operations.
1497
+ load_policy : str, default: "fresh"
1498
+ The policy for loading the checkpoint. The following policies are supported:
1499
+ - "eager": Loads the the latest available checkpoint within the namespace.
1500
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
1501
+ will be loaded at the start of the task.
1502
+ - "none": Do not load any checkpoint
1503
+ - "fresh": Loads the lastest checkpoint created within the running Task.
1504
+ This mode helps loading checkpoints across various retry attempts of the same task.
1505
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
1506
+ created within the task will be loaded when the task is retries execution on failure.
1507
+
1508
+ temp_dir_root : str, default: None
1509
+ The root directory under which `current.checkpoint.directory` will be created.
1510
+ """
1511
+ ...
1512
+
1513
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1514
+ """
1515
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1516
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1517
+
1518
+
1519
+ Parameters
1520
+ ----------
1521
+ timeout : int
1522
+ Time, in seconds before the task times out and fails. (Default: 3600)
1523
+ poke_interval : int
1524
+ Time in seconds that the job should wait in between each try. (Default: 60)
1525
+ mode : str
1526
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1527
+ exponential_backoff : bool
1528
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1529
+ pool : str
1530
+ the slot pool this task should run in,
1531
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1532
+ soft_fail : bool
1533
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1534
+ name : str
1535
+ Name of the sensor on Airflow
1536
+ description : str
1537
+ Description of sensor in the Airflow UI
1538
+ external_dag_id : str
1539
+ The dag_id that contains the task you want to wait for.
1540
+ external_task_ids : List[str]
1541
+ The list of task_ids that you want to wait for.
1542
+ If None (default value) the sensor waits for the DAG. (Default: None)
1543
+ allowed_states : List[str]
1544
+ Iterable of allowed states, (Default: ['success'])
1545
+ failed_states : List[str]
1546
+ Iterable of failed or dis-allowed states. (Default: None)
1547
+ execution_delta : datetime.timedelta
1548
+ time difference with the previous execution to look at,
1549
+ the default is the same logical date as the current task or DAG. (Default: None)
1550
+ check_existence: bool
1551
+ Set to True to check if the external task exists or check if
1552
+ the DAG to wait for exists. (Default: True)
1553
+ """
1554
+ ...
1555
+
1556
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1557
+ """
1558
+ Specifies what flows belong to the same project.
1559
+
1560
+ A project-specific namespace is created for all flows that
1561
+ use the same `@project(name)`.
1562
+
1563
+
1564
+ Parameters
1565
+ ----------
1566
+ name : str
1567
+ Project name. Make sure that the name is unique amongst all
1568
+ projects that use the same production scheduler. The name may
1569
+ contain only lowercase alphanumeric characters and underscores.
1570
+
1571
+ branch : Optional[str], default None
1572
+ The branch to use. If not specified, the branch is set to
1573
+ `user.<username>` unless `production` is set to `True`. This can
1574
+ also be set on the command line using `--branch` as a top-level option.
1575
+ It is an error to specify `branch` in the decorator and on the command line.
1576
+
1577
+ production : bool, default False
1578
+ Whether or not the branch is the production branch. This can also be set on the
1579
+ command line using `--production` as a top-level option. It is an error to specify
1580
+ `production` in the decorator and on the command line.
1581
+ The project branch name will be:
1582
+ - if `branch` is specified:
1583
+ - if `production` is True: `prod.<branch>`
1584
+ - if `production` is False: `test.<branch>`
1585
+ - if `branch` is not specified:
1586
+ - if `production` is True: `prod`
1587
+ - if `production` is False: `user.<username>`
1509
1588
  """
1510
1589
  ...
1511
1590
 
@@ -1601,38 +1680,96 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
1601
1680
  """
1602
1681
  ...
1603
1682
 
1604
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1683
+ @typing.overload
1684
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1605
1685
  """
1606
- Specifies what flows belong to the same project.
1686
+ Specifies the event(s) that this flow depends on.
1607
1687
 
1608
- A project-specific namespace is created for all flows that
1609
- use the same `@project(name)`.
1688
+ ```
1689
+ @trigger(event='foo')
1690
+ ```
1691
+ or
1692
+ ```
1693
+ @trigger(events=['foo', 'bar'])
1694
+ ```
1695
+
1696
+ Additionally, you can specify the parameter mappings
1697
+ to map event payload to Metaflow parameters for the flow.
1698
+ ```
1699
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1700
+ ```
1701
+ or
1702
+ ```
1703
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1704
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1705
+ ```
1706
+
1707
+ 'parameters' can also be a list of strings and tuples like so:
1708
+ ```
1709
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1710
+ ```
1711
+ This is equivalent to:
1712
+ ```
1713
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1714
+ ```
1610
1715
 
1611
1716
 
1612
1717
  Parameters
1613
1718
  ----------
1614
- name : str
1615
- Project name. Make sure that the name is unique amongst all
1616
- projects that use the same production scheduler. The name may
1617
- contain only lowercase alphanumeric characters and underscores.
1719
+ event : Union[str, Dict[str, Any]], optional, default None
1720
+ Event dependency for this flow.
1721
+ events : List[Union[str, Dict[str, Any]]], default []
1722
+ Events dependency for this flow.
1723
+ options : Dict[str, Any], default {}
1724
+ Backend-specific configuration for tuning eventing behavior.
1725
+ """
1726
+ ...
1727
+
1728
+ @typing.overload
1729
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1730
+ ...
1731
+
1732
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1733
+ """
1734
+ Specifies the event(s) that this flow depends on.
1618
1735
 
1619
- branch : Optional[str], default None
1620
- The branch to use. If not specified, the branch is set to
1621
- `user.<username>` unless `production` is set to `True`. This can
1622
- also be set on the command line using `--branch` as a top-level option.
1623
- It is an error to specify `branch` in the decorator and on the command line.
1736
+ ```
1737
+ @trigger(event='foo')
1738
+ ```
1739
+ or
1740
+ ```
1741
+ @trigger(events=['foo', 'bar'])
1742
+ ```
1624
1743
 
1625
- production : bool, default False
1626
- Whether or not the branch is the production branch. This can also be set on the
1627
- command line using `--production` as a top-level option. It is an error to specify
1628
- `production` in the decorator and on the command line.
1629
- The project branch name will be:
1630
- - if `branch` is specified:
1631
- - if `production` is True: `prod.<branch>`
1632
- - if `production` is False: `test.<branch>`
1633
- - if `branch` is not specified:
1634
- - if `production` is True: `prod`
1635
- - if `production` is False: `user.<username>`
1744
+ Additionally, you can specify the parameter mappings
1745
+ to map event payload to Metaflow parameters for the flow.
1746
+ ```
1747
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1748
+ ```
1749
+ or
1750
+ ```
1751
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1752
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1753
+ ```
1754
+
1755
+ 'parameters' can also be a list of strings and tuples like so:
1756
+ ```
1757
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1758
+ ```
1759
+ This is equivalent to:
1760
+ ```
1761
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1762
+ ```
1763
+
1764
+
1765
+ Parameters
1766
+ ----------
1767
+ event : Union[str, Dict[str, Any]], optional, default None
1768
+ Event dependency for this flow.
1769
+ events : List[Union[str, Dict[str, Any]]], default []
1770
+ Events dependency for this flow.
1771
+ options : Dict[str, Any], default {}
1772
+ Backend-specific configuration for tuning eventing behavior.
1636
1773
  """
1637
1774
  ...
1638
1775
 
@@ -1844,49 +1981,6 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1844
1981
  """
1845
1982
  ...
1846
1983
 
1847
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1848
- """
1849
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1850
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1851
-
1852
-
1853
- Parameters
1854
- ----------
1855
- timeout : int
1856
- Time, in seconds before the task times out and fails. (Default: 3600)
1857
- poke_interval : int
1858
- Time in seconds that the job should wait in between each try. (Default: 60)
1859
- mode : str
1860
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1861
- exponential_backoff : bool
1862
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1863
- pool : str
1864
- the slot pool this task should run in,
1865
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1866
- soft_fail : bool
1867
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1868
- name : str
1869
- Name of the sensor on Airflow
1870
- description : str
1871
- Description of sensor in the Airflow UI
1872
- external_dag_id : str
1873
- The dag_id that contains the task you want to wait for.
1874
- external_task_ids : List[str]
1875
- The list of task_ids that you want to wait for.
1876
- If None (default value) the sensor waits for the DAG. (Default: None)
1877
- allowed_states : List[str]
1878
- Iterable of allowed states, (Default: ['success'])
1879
- failed_states : List[str]
1880
- Iterable of failed or dis-allowed states. (Default: None)
1881
- execution_delta : datetime.timedelta
1882
- time difference with the previous execution to look at,
1883
- the default is the same logical date as the current task or DAG. (Default: None)
1884
- check_existence: bool
1885
- Set to True to check if the external task exists or check if
1886
- the DAG to wait for exists. (Default: True)
1887
- """
1888
- ...
1889
-
1890
1984
  @typing.overload
1891
1985
  def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1892
1986
  """
@@ -1988,98 +2082,5 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1988
2082
  """
1989
2083
  ...
1990
2084
 
1991
- @typing.overload
1992
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1993
- """
1994
- Specifies the event(s) that this flow depends on.
1995
-
1996
- ```
1997
- @trigger(event='foo')
1998
- ```
1999
- or
2000
- ```
2001
- @trigger(events=['foo', 'bar'])
2002
- ```
2003
-
2004
- Additionally, you can specify the parameter mappings
2005
- to map event payload to Metaflow parameters for the flow.
2006
- ```
2007
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
2008
- ```
2009
- or
2010
- ```
2011
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
2012
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
2013
- ```
2014
-
2015
- 'parameters' can also be a list of strings and tuples like so:
2016
- ```
2017
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
2018
- ```
2019
- This is equivalent to:
2020
- ```
2021
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
2022
- ```
2023
-
2024
-
2025
- Parameters
2026
- ----------
2027
- event : Union[str, Dict[str, Any]], optional, default None
2028
- Event dependency for this flow.
2029
- events : List[Union[str, Dict[str, Any]]], default []
2030
- Events dependency for this flow.
2031
- options : Dict[str, Any], default {}
2032
- Backend-specific configuration for tuning eventing behavior.
2033
- """
2034
- ...
2035
-
2036
- @typing.overload
2037
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
2038
- ...
2039
-
2040
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
2041
- """
2042
- Specifies the event(s) that this flow depends on.
2043
-
2044
- ```
2045
- @trigger(event='foo')
2046
- ```
2047
- or
2048
- ```
2049
- @trigger(events=['foo', 'bar'])
2050
- ```
2051
-
2052
- Additionally, you can specify the parameter mappings
2053
- to map event payload to Metaflow parameters for the flow.
2054
- ```
2055
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
2056
- ```
2057
- or
2058
- ```
2059
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
2060
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
2061
- ```
2062
-
2063
- 'parameters' can also be a list of strings and tuples like so:
2064
- ```
2065
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
2066
- ```
2067
- This is equivalent to:
2068
- ```
2069
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
2070
- ```
2071
-
2072
-
2073
- Parameters
2074
- ----------
2075
- event : Union[str, Dict[str, Any]], optional, default None
2076
- Event dependency for this flow.
2077
- events : List[Union[str, Dict[str, Any]]], default []
2078
- Events dependency for this flow.
2079
- options : Dict[str, Any], default {}
2080
- Backend-specific configuration for tuning eventing behavior.
2081
- """
2082
- ...
2083
-
2084
2085
  pkg_name: str
2085
2086