ob-metaflow-stubs 6.0.5.2__py2.py3-none-any.whl → 6.0.6.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (261) hide show
  1. metaflow-stubs/__init__.pyi +911 -911
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +38 -38
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +4 -4
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +3 -3
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +2 -2
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +2 -2
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +3 -3
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +3 -3
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +3 -3
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  110. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  111. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  112. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  113. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  114. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
  116. metaflow-stubs/multicore_utils.pyi +2 -2
  117. metaflow-stubs/ob_internal.pyi +2 -2
  118. metaflow-stubs/packaging_sys/__init__.pyi +5 -5
  119. metaflow-stubs/packaging_sys/backend.pyi +3 -3
  120. metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
  121. metaflow-stubs/packaging_sys/tar_backend.pyi +5 -5
  122. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  123. metaflow-stubs/packaging_sys/v1.pyi +2 -2
  124. metaflow-stubs/parameters.pyi +3 -3
  125. metaflow-stubs/plugins/__init__.pyi +13 -13
  126. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  128. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  129. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  130. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  131. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  132. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  133. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  134. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  135. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  136. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  137. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  138. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  139. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  140. metaflow-stubs/plugins/argo/exit_hooks.pyi +2 -2
  141. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  142. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  143. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  144. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  146. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  147. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  148. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  150. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  152. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  154. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  155. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  157. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  158. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  159. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  160. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  161. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  162. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  163. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  164. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  165. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  166. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  167. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  168. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  169. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  170. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  171. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  172. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  173. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  174. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  175. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  176. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  177. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  178. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  179. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  180. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  181. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  182. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  183. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  184. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  185. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  186. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  187. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  188. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  189. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  190. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  191. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  192. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  193. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  194. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  195. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  196. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  197. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  198. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  199. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  200. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  201. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  202. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  203. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  204. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  205. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  206. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  207. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  208. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  209. metaflow-stubs/plugins/perimeters.pyi +2 -2
  210. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  211. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  212. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  213. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  214. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  215. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  216. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  217. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  218. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  219. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  220. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  221. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  222. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  223. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  224. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  225. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  226. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  227. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  228. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  229. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  230. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  231. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  232. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  233. metaflow-stubs/profilers/__init__.pyi +2 -2
  234. metaflow-stubs/pylint_wrapper.pyi +2 -2
  235. metaflow-stubs/runner/__init__.pyi +2 -2
  236. metaflow-stubs/runner/deployer.pyi +32 -32
  237. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  238. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  239. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  240. metaflow-stubs/runner/nbrun.pyi +2 -2
  241. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  242. metaflow-stubs/runner/utils.pyi +3 -3
  243. metaflow-stubs/system/__init__.pyi +2 -2
  244. metaflow-stubs/system/system_logger.pyi +2 -2
  245. metaflow-stubs/system/system_monitor.pyi +2 -2
  246. metaflow-stubs/tagging_util.pyi +2 -2
  247. metaflow-stubs/tuple_util.pyi +2 -2
  248. metaflow-stubs/user_configs/__init__.pyi +2 -2
  249. metaflow-stubs/user_configs/config_options.pyi +2 -2
  250. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  251. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  252. metaflow-stubs/user_decorators/common.pyi +2 -2
  253. metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
  254. metaflow-stubs/user_decorators/mutable_step.pyi +6 -6
  255. metaflow-stubs/user_decorators/user_flow_decorator.pyi +4 -4
  256. metaflow-stubs/user_decorators/user_step_decorator.pyi +4 -4
  257. {ob_metaflow_stubs-6.0.5.2.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/METADATA +1 -1
  258. ob_metaflow_stubs-6.0.6.0.dist-info/RECORD +261 -0
  259. ob_metaflow_stubs-6.0.5.2.dist-info/RECORD +0 -261
  260. {ob_metaflow_stubs-6.0.5.2.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/WHEEL +0 -0
  261. {ob_metaflow_stubs-6.0.5.2.dist-info → ob_metaflow_stubs-6.0.6.0.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.16.8.1+obcheckpoint(0.2.4);ob(v1) #
4
- # Generated on 2025-08-04T19:06:54.653206 #
3
+ # MF version: 2.17.0.1+obcheckpoint(0.2.4);ob(v1) #
4
+ # Generated on 2025-08-05T23:30:10.180789 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -39,9 +39,9 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
39
39
  from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
+ from . import metaflow_git as metaflow_git
42
43
  from . import cards as cards
43
44
  from . import tuple_util as tuple_util
44
- from . import metaflow_git as metaflow_git
45
45
  from . import events as events
46
46
  from . import runner as runner
47
47
  from . import plugins as plugins
@@ -49,8 +49,8 @@ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package imp
49
49
  from . import includefile as includefile
50
50
  from .includefile import IncludeFile as IncludeFile
51
51
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
52
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
53
52
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
53
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
54
54
  from . import client as client
55
55
  from .client.core import namespace as namespace
56
56
  from .client.core import get_namespace as get_namespace
@@ -167,504 +167,386 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
167
167
  """
168
168
  ...
169
169
 
170
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
170
+ @typing.overload
171
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
171
172
  """
172
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
173
-
174
- User code call
175
- --------------
176
- @ollama(
177
- models=[...],
178
- ...
179
- )
180
-
181
- Valid backend options
182
- ---------------------
183
- - 'local': Run as a separate process on the local task machine.
184
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
185
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
186
-
187
- Valid model options
188
- -------------------
189
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
173
+ Specifies environment variables to be set prior to the execution of a step.
190
174
 
191
175
 
192
176
  Parameters
193
177
  ----------
194
- models: list[str]
195
- List of Ollama containers running models in sidecars.
196
- backend: str
197
- Determines where and how to run the Ollama process.
198
- force_pull: bool
199
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
200
- cache_update_policy: str
201
- Cache update policy: "auto", "force", or "never".
202
- force_cache_update: bool
203
- Simple override for "force" cache update policy.
204
- debug: bool
205
- Whether to turn on verbose debugging logs.
206
- circuit_breaker_config: dict
207
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
208
- timeout_config: dict
209
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
178
+ vars : Dict[str, str], default {}
179
+ Dictionary of environment variables to set.
210
180
  """
211
181
  ...
212
182
 
213
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
183
+ @typing.overload
184
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
185
+ ...
186
+
187
+ @typing.overload
188
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
189
+ ...
190
+
191
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
214
192
  """
215
- S3 Proxy decorator for routing S3 requests through a local proxy service.
193
+ Specifies environment variables to be set prior to the execution of a step.
216
194
 
217
195
 
218
196
  Parameters
219
197
  ----------
220
- integration_name : str, optional
221
- Name of the S3 proxy integration. If not specified, will use the only
222
- available S3 proxy integration in the namespace (fails if multiple exist).
223
- write_mode : str, optional
224
- The desired behavior during write operations to target (origin) S3 bucket.
225
- allowed options are:
226
- "origin-and-cache" -> write to both the target S3 bucket and local object
227
- storage
228
- "origin" -> only write to the target S3 bucket
229
- "cache" -> only write to the object storage service used for caching
230
- debug : bool, optional
231
- Enable debug logging for proxy operations.
198
+ vars : Dict[str, str], default {}
199
+ Dictionary of environment variables to set.
232
200
  """
233
201
  ...
234
202
 
235
203
  @typing.overload
236
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
204
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
237
205
  """
238
- Enables checkpointing for a step.
239
-
240
- > Examples
241
-
242
- - Saving Checkpoints
243
-
244
- ```python
245
- @checkpoint
246
- @step
247
- def train(self):
248
- model = create_model(self.parameters, checkpoint_path = None)
249
- for i in range(self.epochs):
250
- # some training logic
251
- loss = model.train(self.dataset)
252
- if i % 10 == 0:
253
- model.save(
254
- current.checkpoint.directory,
255
- )
256
- # saves the contents of the `current.checkpoint.directory` as a checkpoint
257
- # and returns a reference dictionary to the checkpoint saved in the datastore
258
- self.latest_checkpoint = current.checkpoint.save(
259
- name="epoch_checkpoint",
260
- metadata={
261
- "epoch": i,
262
- "loss": loss,
263
- }
264
- )
265
- ```
266
-
267
- - Using Loaded Checkpoints
206
+ Internal decorator to support Fast bakery
207
+ """
208
+ ...
209
+
210
+ @typing.overload
211
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
212
+ ...
213
+
214
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
215
+ """
216
+ Internal decorator to support Fast bakery
217
+ """
218
+ ...
219
+
220
+ @typing.overload
221
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
222
+ """
223
+ Specifies the number of times the task corresponding
224
+ to a step needs to be retried.
268
225
 
269
- ```python
270
- @retry(times=3)
271
- @checkpoint
272
- @step
273
- def train(self):
274
- # Assume that the task has restarted and the previous attempt of the task
275
- # saved a checkpoint
276
- checkpoint_path = None
277
- if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
278
- print("Loaded checkpoint from the previous attempt")
279
- checkpoint_path = current.checkpoint.directory
226
+ This decorator is useful for handling transient errors, such as networking issues.
227
+ If your task contains operations that can't be retried safely, e.g. database updates,
228
+ it is advisable to annotate it with `@retry(times=0)`.
280
229
 
281
- model = create_model(self.parameters, checkpoint_path = checkpoint_path)
282
- for i in range(self.epochs):
283
- ...
284
- ```
230
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
231
+ decorator will execute a no-op task after all retries have been exhausted,
232
+ ensuring that the flow execution can continue.
285
233
 
286
234
 
287
235
  Parameters
288
236
  ----------
289
- load_policy : str, default: "fresh"
290
- The policy for loading the checkpoint. The following policies are supported:
291
- - "eager": Loads the the latest available checkpoint within the namespace.
292
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
293
- will be loaded at the start of the task.
294
- - "none": Do not load any checkpoint
295
- - "fresh": Loads the lastest checkpoint created within the running Task.
296
- This mode helps loading checkpoints across various retry attempts of the same task.
297
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
298
- created within the task will be loaded when the task is retries execution on failure.
299
-
300
- temp_dir_root : str, default: None
301
- The root directory under which `current.checkpoint.directory` will be created.
237
+ times : int, default 3
238
+ Number of times to retry this task.
239
+ minutes_between_retries : int, default 2
240
+ Number of minutes between retries.
302
241
  """
303
242
  ...
304
243
 
305
244
  @typing.overload
306
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
245
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
307
246
  ...
308
247
 
309
248
  @typing.overload
310
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
249
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
311
250
  ...
312
251
 
313
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
252
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
314
253
  """
315
- Enables checkpointing for a step.
316
-
317
- > Examples
318
-
319
- - Saving Checkpoints
320
-
321
- ```python
322
- @checkpoint
323
- @step
324
- def train(self):
325
- model = create_model(self.parameters, checkpoint_path = None)
326
- for i in range(self.epochs):
327
- # some training logic
328
- loss = model.train(self.dataset)
329
- if i % 10 == 0:
330
- model.save(
331
- current.checkpoint.directory,
332
- )
333
- # saves the contents of the `current.checkpoint.directory` as a checkpoint
334
- # and returns a reference dictionary to the checkpoint saved in the datastore
335
- self.latest_checkpoint = current.checkpoint.save(
336
- name="epoch_checkpoint",
337
- metadata={
338
- "epoch": i,
339
- "loss": loss,
340
- }
341
- )
342
- ```
343
-
344
- - Using Loaded Checkpoints
254
+ Specifies the number of times the task corresponding
255
+ to a step needs to be retried.
345
256
 
346
- ```python
347
- @retry(times=3)
348
- @checkpoint
349
- @step
350
- def train(self):
351
- # Assume that the task has restarted and the previous attempt of the task
352
- # saved a checkpoint
353
- checkpoint_path = None
354
- if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
355
- print("Loaded checkpoint from the previous attempt")
356
- checkpoint_path = current.checkpoint.directory
257
+ This decorator is useful for handling transient errors, such as networking issues.
258
+ If your task contains operations that can't be retried safely, e.g. database updates,
259
+ it is advisable to annotate it with `@retry(times=0)`.
357
260
 
358
- model = create_model(self.parameters, checkpoint_path = checkpoint_path)
359
- for i in range(self.epochs):
360
- ...
361
- ```
261
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
262
+ decorator will execute a no-op task after all retries have been exhausted,
263
+ ensuring that the flow execution can continue.
362
264
 
363
265
 
364
266
  Parameters
365
267
  ----------
366
- load_policy : str, default: "fresh"
367
- The policy for loading the checkpoint. The following policies are supported:
368
- - "eager": Loads the the latest available checkpoint within the namespace.
369
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
370
- will be loaded at the start of the task.
371
- - "none": Do not load any checkpoint
372
- - "fresh": Loads the lastest checkpoint created within the running Task.
373
- This mode helps loading checkpoints across various retry attempts of the same task.
374
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
375
- created within the task will be loaded when the task is retries execution on failure.
376
-
377
- temp_dir_root : str, default: None
378
- The root directory under which `current.checkpoint.directory` will be created.
268
+ times : int, default 3
269
+ Number of times to retry this task.
270
+ minutes_between_retries : int, default 2
271
+ Number of minutes between retries.
379
272
  """
380
273
  ...
381
274
 
382
275
  @typing.overload
383
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
276
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
384
277
  """
385
- Specifies environment variables to be set prior to the execution of a step.
278
+ Specifies the PyPI packages for the step.
279
+
280
+ Information in this decorator will augment any
281
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
282
+ you can use `@pypi_base` to set packages required by all
283
+ steps and use `@pypi` to specify step-specific overrides.
386
284
 
387
285
 
388
286
  Parameters
389
287
  ----------
390
- vars : Dict[str, str], default {}
391
- Dictionary of environment variables to set.
288
+ packages : Dict[str, str], default: {}
289
+ Packages to use for this step. The key is the name of the package
290
+ and the value is the version to use.
291
+ python : str, optional, default: None
292
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
293
+ that the version used will correspond to the version of the Python interpreter used to start the run.
392
294
  """
393
295
  ...
394
296
 
395
297
  @typing.overload
396
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
298
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
397
299
  ...
398
300
 
399
301
  @typing.overload
400
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
302
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
401
303
  ...
402
304
 
403
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
404
- """
405
- Specifies environment variables to be set prior to the execution of a step.
406
-
407
-
408
- Parameters
409
- ----------
410
- vars : Dict[str, str], default {}
411
- Dictionary of environment variables to set.
412
- """
413
- ...
414
-
415
- @typing.overload
416
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
305
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
417
306
  """
418
- Specifies that the step will success under all circumstances.
307
+ Specifies the PyPI packages for the step.
419
308
 
420
- The decorator will create an optional artifact, specified by `var`, which
421
- contains the exception raised. You can use it to detect the presence
422
- of errors, indicating that all happy-path artifacts produced by the step
423
- are missing.
309
+ Information in this decorator will augment any
310
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
311
+ you can use `@pypi_base` to set packages required by all
312
+ steps and use `@pypi` to specify step-specific overrides.
424
313
 
425
314
 
426
315
  Parameters
427
316
  ----------
428
- var : str, optional, default None
429
- Name of the artifact in which to store the caught exception.
430
- If not specified, the exception is not stored.
431
- print_exception : bool, default True
432
- Determines whether or not the exception is printed to
433
- stdout when caught.
317
+ packages : Dict[str, str], default: {}
318
+ Packages to use for this step. The key is the name of the package
319
+ and the value is the version to use.
320
+ python : str, optional, default: None
321
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
322
+ that the version used will correspond to the version of the Python interpreter used to start the run.
434
323
  """
435
324
  ...
436
325
 
437
326
  @typing.overload
438
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
439
- ...
440
-
441
- @typing.overload
442
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
443
- ...
444
-
445
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
446
- """
447
- Specifies that the step will success under all circumstances.
448
-
449
- The decorator will create an optional artifact, specified by `var`, which
450
- contains the exception raised. You can use it to detect the presence
451
- of errors, indicating that all happy-path artifacts produced by the step
452
- are missing.
453
-
454
-
455
- Parameters
456
- ----------
457
- var : str, optional, default None
458
- Name of the artifact in which to store the caught exception.
459
- If not specified, the exception is not stored.
460
- print_exception : bool, default True
461
- Determines whether or not the exception is printed to
462
- stdout when caught.
463
- """
464
- ...
465
-
466
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
327
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
467
328
  """
468
- Specifies that this step should execute on DGX cloud.
329
+ Specifies a timeout for your step.
469
330
 
331
+ This decorator is useful if this step may hang indefinitely.
470
332
 
471
- Parameters
472
- ----------
473
- gpu : int
474
- Number of GPUs to use.
475
- gpu_type : str
476
- Type of Nvidia GPU to use.
477
- queue_timeout : int
478
- Time to keep the job in NVCF's queue.
479
- """
480
- ...
481
-
482
- @typing.overload
483
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
484
- """
485
- Creates a human-readable report, a Metaflow Card, after this step completes.
333
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
334
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
335
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
486
336
 
487
- Note that you may add multiple `@card` decorators in a step with different parameters.
337
+ Note that all the values specified in parameters are added together so if you specify
338
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
488
339
 
489
340
 
490
341
  Parameters
491
342
  ----------
492
- type : str, default 'default'
493
- Card type.
494
- id : str, optional, default None
495
- If multiple cards are present, use this id to identify this card.
496
- options : Dict[str, Any], default {}
497
- Options passed to the card. The contents depend on the card type.
498
- timeout : int, default 45
499
- Interrupt reporting if it takes more than this many seconds.
343
+ seconds : int, default 0
344
+ Number of seconds to wait prior to timing out.
345
+ minutes : int, default 0
346
+ Number of minutes to wait prior to timing out.
347
+ hours : int, default 0
348
+ Number of hours to wait prior to timing out.
500
349
  """
501
350
  ...
502
351
 
503
352
  @typing.overload
504
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
353
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
505
354
  ...
506
355
 
507
356
  @typing.overload
508
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
357
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
509
358
  ...
510
359
 
511
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
360
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
512
361
  """
513
- Creates a human-readable report, a Metaflow Card, after this step completes.
362
+ Specifies a timeout for your step.
514
363
 
515
- Note that you may add multiple `@card` decorators in a step with different parameters.
364
+ This decorator is useful if this step may hang indefinitely.
365
+
366
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
367
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
368
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
369
+
370
+ Note that all the values specified in parameters are added together so if you specify
371
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
516
372
 
517
373
 
518
374
  Parameters
519
375
  ----------
520
- type : str, default 'default'
521
- Card type.
522
- id : str, optional, default None
523
- If multiple cards are present, use this id to identify this card.
524
- options : Dict[str, Any], default {}
525
- Options passed to the card. The contents depend on the card type.
526
- timeout : int, default 45
527
- Interrupt reporting if it takes more than this many seconds.
376
+ seconds : int, default 0
377
+ Number of seconds to wait prior to timing out.
378
+ minutes : int, default 0
379
+ Number of minutes to wait prior to timing out.
380
+ hours : int, default 0
381
+ Number of hours to wait prior to timing out.
528
382
  """
529
383
  ...
530
384
 
531
385
  @typing.overload
532
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
386
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
533
387
  """
534
- Internal decorator to support Fast bakery
388
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
389
+ It exists to make it easier for users to know that this decorator should only be used with
390
+ a Neo Cloud like Nebius.
535
391
  """
536
392
  ...
537
393
 
538
394
  @typing.overload
539
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
395
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
540
396
  ...
541
397
 
542
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
398
+ def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
543
399
  """
544
- Internal decorator to support Fast bakery
400
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
401
+ It exists to make it easier for users to know that this decorator should only be used with
402
+ a Neo Cloud like Nebius.
545
403
  """
546
404
  ...
547
405
 
548
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
406
+ @typing.overload
407
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
549
408
  """
550
- Specifies that this step should execute on DGX cloud.
409
+ Enables checkpointing for a step.
551
410
 
411
+ > Examples
552
412
 
553
- Parameters
554
- ----------
555
- gpu : int
556
- Number of GPUs to use.
557
- gpu_type : str
558
- Type of Nvidia GPU to use.
559
- """
560
- ...
561
-
562
- @typing.overload
563
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
564
- """
565
- Specifies the PyPI packages for the step.
413
+ - Saving Checkpoints
566
414
 
567
- Information in this decorator will augment any
568
- attributes set in the `@pyi_base` flow-level decorator. Hence,
569
- you can use `@pypi_base` to set packages required by all
570
- steps and use `@pypi` to specify step-specific overrides.
415
+ ```python
416
+ @checkpoint
417
+ @step
418
+ def train(self):
419
+ model = create_model(self.parameters, checkpoint_path = None)
420
+ for i in range(self.epochs):
421
+ # some training logic
422
+ loss = model.train(self.dataset)
423
+ if i % 10 == 0:
424
+ model.save(
425
+ current.checkpoint.directory,
426
+ )
427
+ # saves the contents of the `current.checkpoint.directory` as a checkpoint
428
+ # and returns a reference dictionary to the checkpoint saved in the datastore
429
+ self.latest_checkpoint = current.checkpoint.save(
430
+ name="epoch_checkpoint",
431
+ metadata={
432
+ "epoch": i,
433
+ "loss": loss,
434
+ }
435
+ )
436
+ ```
437
+
438
+ - Using Loaded Checkpoints
439
+
440
+ ```python
441
+ @retry(times=3)
442
+ @checkpoint
443
+ @step
444
+ def train(self):
445
+ # Assume that the task has restarted and the previous attempt of the task
446
+ # saved a checkpoint
447
+ checkpoint_path = None
448
+ if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
449
+ print("Loaded checkpoint from the previous attempt")
450
+ checkpoint_path = current.checkpoint.directory
451
+
452
+ model = create_model(self.parameters, checkpoint_path = checkpoint_path)
453
+ for i in range(self.epochs):
454
+ ...
455
+ ```
571
456
 
572
457
 
573
458
  Parameters
574
459
  ----------
575
- packages : Dict[str, str], default: {}
576
- Packages to use for this step. The key is the name of the package
577
- and the value is the version to use.
578
- python : str, optional, default: None
579
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
580
- that the version used will correspond to the version of the Python interpreter used to start the run.
460
+ load_policy : str, default: "fresh"
461
+ The policy for loading the checkpoint. The following policies are supported:
462
+ - "eager": Loads the the latest available checkpoint within the namespace.
463
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
464
+ will be loaded at the start of the task.
465
+ - "none": Do not load any checkpoint
466
+ - "fresh": Loads the lastest checkpoint created within the running Task.
467
+ This mode helps loading checkpoints across various retry attempts of the same task.
468
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
469
+ created within the task will be loaded when the task is retries execution on failure.
470
+
471
+ temp_dir_root : str, default: None
472
+ The root directory under which `current.checkpoint.directory` will be created.
581
473
  """
582
474
  ...
583
475
 
584
476
  @typing.overload
585
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
477
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
586
478
  ...
587
479
 
588
480
  @typing.overload
589
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
481
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
590
482
  ...
591
483
 
592
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
484
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
593
485
  """
594
- Specifies the PyPI packages for the step.
486
+ Enables checkpointing for a step.
595
487
 
596
- Information in this decorator will augment any
597
- attributes set in the `@pyi_base` flow-level decorator. Hence,
598
- you can use `@pypi_base` to set packages required by all
599
- steps and use `@pypi` to specify step-specific overrides.
488
+ > Examples
600
489
 
490
+ - Saving Checkpoints
601
491
 
602
- Parameters
603
- ----------
604
- packages : Dict[str, str], default: {}
605
- Packages to use for this step. The key is the name of the package
606
- and the value is the version to use.
607
- python : str, optional, default: None
608
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
609
- that the version used will correspond to the version of the Python interpreter used to start the run.
610
- """
611
- ...
612
-
613
- @typing.overload
614
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
615
- """
616
- Specifies secrets to be retrieved and injected as environment variables prior to
617
- the execution of a step.
492
+ ```python
493
+ @checkpoint
494
+ @step
495
+ def train(self):
496
+ model = create_model(self.parameters, checkpoint_path = None)
497
+ for i in range(self.epochs):
498
+ # some training logic
499
+ loss = model.train(self.dataset)
500
+ if i % 10 == 0:
501
+ model.save(
502
+ current.checkpoint.directory,
503
+ )
504
+ # saves the contents of the `current.checkpoint.directory` as a checkpoint
505
+ # and returns a reference dictionary to the checkpoint saved in the datastore
506
+ self.latest_checkpoint = current.checkpoint.save(
507
+ name="epoch_checkpoint",
508
+ metadata={
509
+ "epoch": i,
510
+ "loss": loss,
511
+ }
512
+ )
513
+ ```
618
514
 
515
+ - Using Loaded Checkpoints
619
516
 
620
- Parameters
621
- ----------
622
- sources : List[Union[str, Dict[str, Any]]], default: []
623
- List of secret specs, defining how the secrets are to be retrieved
624
- role : str, optional, default: None
625
- Role to use for fetching secrets
626
- """
627
- ...
628
-
629
- @typing.overload
630
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
631
- ...
632
-
633
- @typing.overload
634
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
635
- ...
636
-
637
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
638
- """
639
- Specifies secrets to be retrieved and injected as environment variables prior to
640
- the execution of a step.
517
+ ```python
518
+ @retry(times=3)
519
+ @checkpoint
520
+ @step
521
+ def train(self):
522
+ # Assume that the task has restarted and the previous attempt of the task
523
+ # saved a checkpoint
524
+ checkpoint_path = None
525
+ if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
526
+ print("Loaded checkpoint from the previous attempt")
527
+ checkpoint_path = current.checkpoint.directory
528
+
529
+ model = create_model(self.parameters, checkpoint_path = checkpoint_path)
530
+ for i in range(self.epochs):
531
+ ...
532
+ ```
641
533
 
642
534
 
643
535
  Parameters
644
536
  ----------
645
- sources : List[Union[str, Dict[str, Any]]], default: []
646
- List of secret specs, defining how the secrets are to be retrieved
647
- role : str, optional, default: None
648
- Role to use for fetching secrets
649
- """
650
- ...
651
-
652
- @typing.overload
653
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
654
- """
655
- A simple decorator that demonstrates using CardDecoratorInjector
656
- to inject a card and render simple markdown content.
657
- """
658
- ...
659
-
660
- @typing.overload
661
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
662
- ...
663
-
664
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
665
- """
666
- A simple decorator that demonstrates using CardDecoratorInjector
667
- to inject a card and render simple markdown content.
537
+ load_policy : str, default: "fresh"
538
+ The policy for loading the checkpoint. The following policies are supported:
539
+ - "eager": Loads the the latest available checkpoint within the namespace.
540
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
541
+ will be loaded at the start of the task.
542
+ - "none": Do not load any checkpoint
543
+ - "fresh": Loads the lastest checkpoint created within the running Task.
544
+ This mode helps loading checkpoints across various retry attempts of the same task.
545
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
546
+ created within the task will be loaded when the task is retries execution on failure.
547
+
548
+ temp_dir_root : str, default: None
549
+ The root directory under which `current.checkpoint.directory` will be created.
668
550
  """
669
551
  ...
670
552
 
@@ -757,82 +639,6 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
757
639
  """
758
640
  ...
759
641
 
760
- @typing.overload
761
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
762
- """
763
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
764
- It exists to make it easier for users to know that this decorator should only be used with
765
- a Neo Cloud like Nebius.
766
- """
767
- ...
768
-
769
- @typing.overload
770
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
771
- ...
772
-
773
- def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
774
- """
775
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
776
- It exists to make it easier for users to know that this decorator should only be used with
777
- a Neo Cloud like Nebius.
778
- """
779
- ...
780
-
781
- @typing.overload
782
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
783
- """
784
- Specifies the number of times the task corresponding
785
- to a step needs to be retried.
786
-
787
- This decorator is useful for handling transient errors, such as networking issues.
788
- If your task contains operations that can't be retried safely, e.g. database updates,
789
- it is advisable to annotate it with `@retry(times=0)`.
790
-
791
- This can be used in conjunction with the `@catch` decorator. The `@catch`
792
- decorator will execute a no-op task after all retries have been exhausted,
793
- ensuring that the flow execution can continue.
794
-
795
-
796
- Parameters
797
- ----------
798
- times : int, default 3
799
- Number of times to retry this task.
800
- minutes_between_retries : int, default 2
801
- Number of minutes between retries.
802
- """
803
- ...
804
-
805
- @typing.overload
806
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
807
- ...
808
-
809
- @typing.overload
810
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
811
- ...
812
-
813
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
814
- """
815
- Specifies the number of times the task corresponding
816
- to a step needs to be retried.
817
-
818
- This decorator is useful for handling transient errors, such as networking issues.
819
- If your task contains operations that can't be retried safely, e.g. database updates,
820
- it is advisable to annotate it with `@retry(times=0)`.
821
-
822
- This can be used in conjunction with the `@catch` decorator. The `@catch`
823
- decorator will execute a no-op task after all retries have been exhausted,
824
- ensuring that the flow execution can continue.
825
-
826
-
827
- Parameters
828
- ----------
829
- times : int, default 3
830
- Number of times to retry this task.
831
- minutes_between_retries : int, default 2
832
- Number of minutes between retries.
833
- """
834
- ...
835
-
836
642
  def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
837
643
  """
838
644
  Decorator that helps cache, version and store models/datasets from huggingface hub.
@@ -914,121 +720,103 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
914
720
  ...
915
721
 
916
722
  @typing.overload
917
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
723
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
918
724
  """
919
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
920
- It exists to make it easier for users to know that this decorator should only be used with
921
- a Neo Cloud like CoreWeave.
725
+ Decorator prototype for all step decorators. This function gets specialized
726
+ and imported for all decorators types by _import_plugin_decorators().
922
727
  """
923
728
  ...
924
729
 
925
730
  @typing.overload
926
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
731
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
927
732
  ...
928
733
 
929
- def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
734
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
930
735
  """
931
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
932
- It exists to make it easier for users to know that this decorator should only be used with
933
- a Neo Cloud like CoreWeave.
736
+ Decorator prototype for all step decorators. This function gets specialized
737
+ and imported for all decorators types by _import_plugin_decorators().
934
738
  """
935
739
  ...
936
740
 
937
741
  @typing.overload
938
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
742
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
939
743
  """
940
- Specifies the resources needed when executing this step.
941
-
942
- Use `@resources` to specify the resource requirements
943
- independently of the specific compute layer (`@batch`, `@kubernetes`).
744
+ Creates a human-readable report, a Metaflow Card, after this step completes.
944
745
 
945
- You can choose the compute layer on the command line by executing e.g.
946
- ```
947
- python myflow.py run --with batch
948
- ```
949
- or
950
- ```
951
- python myflow.py run --with kubernetes
952
- ```
953
- which executes the flow on the desired system using the
954
- requirements specified in `@resources`.
746
+ Note that you may add multiple `@card` decorators in a step with different parameters.
955
747
 
956
748
 
957
749
  Parameters
958
750
  ----------
959
- cpu : int, default 1
960
- Number of CPUs required for this step.
961
- gpu : int, optional, default None
962
- Number of GPUs required for this step.
963
- disk : int, optional, default None
964
- Disk size (in MB) required for this step. Only applies on Kubernetes.
965
- memory : int, default 4096
966
- Memory size (in MB) required for this step.
967
- shared_memory : int, optional, default None
968
- The value for the size (in MiB) of the /dev/shm volume for this step.
969
- This parameter maps to the `--shm-size` option in Docker.
751
+ type : str, default 'default'
752
+ Card type.
753
+ id : str, optional, default None
754
+ If multiple cards are present, use this id to identify this card.
755
+ options : Dict[str, Any], default {}
756
+ Options passed to the card. The contents depend on the card type.
757
+ timeout : int, default 45
758
+ Interrupt reporting if it takes more than this many seconds.
970
759
  """
971
760
  ...
972
761
 
973
762
  @typing.overload
974
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
763
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
975
764
  ...
976
765
 
977
766
  @typing.overload
978
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
767
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
979
768
  ...
980
769
 
981
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
770
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
982
771
  """
983
- Specifies the resources needed when executing this step.
984
-
985
- Use `@resources` to specify the resource requirements
986
- independently of the specific compute layer (`@batch`, `@kubernetes`).
772
+ Creates a human-readable report, a Metaflow Card, after this step completes.
987
773
 
988
- You can choose the compute layer on the command line by executing e.g.
989
- ```
990
- python myflow.py run --with batch
991
- ```
992
- or
993
- ```
994
- python myflow.py run --with kubernetes
995
- ```
996
- which executes the flow on the desired system using the
997
- requirements specified in `@resources`.
774
+ Note that you may add multiple `@card` decorators in a step with different parameters.
998
775
 
999
776
 
1000
777
  Parameters
1001
778
  ----------
1002
- cpu : int, default 1
1003
- Number of CPUs required for this step.
1004
- gpu : int, optional, default None
1005
- Number of GPUs required for this step.
1006
- disk : int, optional, default None
1007
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1008
- memory : int, default 4096
1009
- Memory size (in MB) required for this step.
1010
- shared_memory : int, optional, default None
1011
- The value for the size (in MiB) of the /dev/shm volume for this step.
1012
- This parameter maps to the `--shm-size` option in Docker.
779
+ type : str, default 'default'
780
+ Card type.
781
+ id : str, optional, default None
782
+ If multiple cards are present, use this id to identify this card.
783
+ options : Dict[str, Any], default {}
784
+ Options passed to the card. The contents depend on the card type.
785
+ timeout : int, default 45
786
+ Interrupt reporting if it takes more than this many seconds.
1013
787
  """
1014
788
  ...
1015
789
 
1016
790
  @typing.overload
1017
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
791
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1018
792
  """
1019
- Decorator prototype for all step decorators. This function gets specialized
1020
- and imported for all decorators types by _import_plugin_decorators().
793
+ A simple decorator that demonstrates using CardDecoratorInjector
794
+ to inject a card and render simple markdown content.
1021
795
  """
1022
796
  ...
1023
797
 
1024
798
  @typing.overload
1025
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
799
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1026
800
  ...
1027
801
 
1028
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
802
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1029
803
  """
1030
- Decorator prototype for all step decorators. This function gets specialized
1031
- and imported for all decorators types by _import_plugin_decorators().
804
+ A simple decorator that demonstrates using CardDecoratorInjector
805
+ to inject a card and render simple markdown content.
806
+ """
807
+ ...
808
+
809
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
810
+ """
811
+ Specifies that this step should execute on DGX cloud.
812
+
813
+
814
+ Parameters
815
+ ----------
816
+ gpu : int
817
+ Number of GPUs to use.
818
+ gpu_type : str
819
+ Type of Nvidia GPU to use.
1032
820
  """
1033
821
  ...
1034
822
 
@@ -1132,100 +920,583 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1132
920
  self.next(self.end)
1133
921
  ```
1134
922
 
1135
- - Loading models
1136
- ```python
1137
- @step
1138
- def train(self):
1139
- # current.model.load returns the path to the model loaded
1140
- checkpoint_path = current.model.load(
1141
- self.checkpoint_key,
1142
- )
1143
- model_path = current.model.load(
1144
- self.model,
1145
- )
1146
- self.next(self.test)
923
+ - Loading models
924
+ ```python
925
+ @step
926
+ def train(self):
927
+ # current.model.load returns the path to the model loaded
928
+ checkpoint_path = current.model.load(
929
+ self.checkpoint_key,
930
+ )
931
+ model_path = current.model.load(
932
+ self.model,
933
+ )
934
+ self.next(self.test)
935
+ ```
936
+
937
+
938
+ Parameters
939
+ ----------
940
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
941
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
942
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
943
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
944
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
945
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
946
+
947
+ temp_dir_root : str, default: None
948
+ The root directory under which `current.model.loaded` will store loaded models
949
+ """
950
+ ...
951
+
952
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
953
+ """
954
+ Specifies that this step should execute on DGX cloud.
955
+
956
+
957
+ Parameters
958
+ ----------
959
+ gpu : int
960
+ Number of GPUs to use.
961
+ gpu_type : str
962
+ Type of Nvidia GPU to use.
963
+ queue_timeout : int
964
+ Time to keep the job in NVCF's queue.
965
+ """
966
+ ...
967
+
968
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
969
+ """
970
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
971
+
972
+ User code call
973
+ --------------
974
+ @ollama(
975
+ models=[...],
976
+ ...
977
+ )
978
+
979
+ Valid backend options
980
+ ---------------------
981
+ - 'local': Run as a separate process on the local task machine.
982
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
983
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
984
+
985
+ Valid model options
986
+ -------------------
987
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
988
+
989
+
990
+ Parameters
991
+ ----------
992
+ models: list[str]
993
+ List of Ollama containers running models in sidecars.
994
+ backend: str
995
+ Determines where and how to run the Ollama process.
996
+ force_pull: bool
997
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
998
+ cache_update_policy: str
999
+ Cache update policy: "auto", "force", or "never".
1000
+ force_cache_update: bool
1001
+ Simple override for "force" cache update policy.
1002
+ debug: bool
1003
+ Whether to turn on verbose debugging logs.
1004
+ circuit_breaker_config: dict
1005
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
1006
+ timeout_config: dict
1007
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
1008
+ """
1009
+ ...
1010
+
1011
+ @typing.overload
1012
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1013
+ """
1014
+ Specifies the Conda environment for the step.
1015
+
1016
+ Information in this decorator will augment any
1017
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1018
+ you can use `@conda_base` to set packages required by all
1019
+ steps and use `@conda` to specify step-specific overrides.
1020
+
1021
+
1022
+ Parameters
1023
+ ----------
1024
+ packages : Dict[str, str], default {}
1025
+ Packages to use for this step. The key is the name of the package
1026
+ and the value is the version to use.
1027
+ libraries : Dict[str, str], default {}
1028
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1029
+ python : str, optional, default None
1030
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1031
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1032
+ disabled : bool, default False
1033
+ If set to True, disables @conda.
1034
+ """
1035
+ ...
1036
+
1037
+ @typing.overload
1038
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1039
+ ...
1040
+
1041
+ @typing.overload
1042
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1043
+ ...
1044
+
1045
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1046
+ """
1047
+ Specifies the Conda environment for the step.
1048
+
1049
+ Information in this decorator will augment any
1050
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1051
+ you can use `@conda_base` to set packages required by all
1052
+ steps and use `@conda` to specify step-specific overrides.
1053
+
1054
+
1055
+ Parameters
1056
+ ----------
1057
+ packages : Dict[str, str], default {}
1058
+ Packages to use for this step. The key is the name of the package
1059
+ and the value is the version to use.
1060
+ libraries : Dict[str, str], default {}
1061
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1062
+ python : str, optional, default None
1063
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1064
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1065
+ disabled : bool, default False
1066
+ If set to True, disables @conda.
1067
+ """
1068
+ ...
1069
+
1070
+ @typing.overload
1071
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1072
+ """
1073
+ Decorator prototype for all step decorators. This function gets specialized
1074
+ and imported for all decorators types by _import_plugin_decorators().
1075
+ """
1076
+ ...
1077
+
1078
+ @typing.overload
1079
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1080
+ ...
1081
+
1082
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1083
+ """
1084
+ Decorator prototype for all step decorators. This function gets specialized
1085
+ and imported for all decorators types by _import_plugin_decorators().
1086
+ """
1087
+ ...
1088
+
1089
+ @typing.overload
1090
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1091
+ """
1092
+ Specifies the resources needed when executing this step.
1093
+
1094
+ Use `@resources` to specify the resource requirements
1095
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1096
+
1097
+ You can choose the compute layer on the command line by executing e.g.
1098
+ ```
1099
+ python myflow.py run --with batch
1100
+ ```
1101
+ or
1102
+ ```
1103
+ python myflow.py run --with kubernetes
1104
+ ```
1105
+ which executes the flow on the desired system using the
1106
+ requirements specified in `@resources`.
1107
+
1108
+
1109
+ Parameters
1110
+ ----------
1111
+ cpu : int, default 1
1112
+ Number of CPUs required for this step.
1113
+ gpu : int, optional, default None
1114
+ Number of GPUs required for this step.
1115
+ disk : int, optional, default None
1116
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1117
+ memory : int, default 4096
1118
+ Memory size (in MB) required for this step.
1119
+ shared_memory : int, optional, default None
1120
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1121
+ This parameter maps to the `--shm-size` option in Docker.
1122
+ """
1123
+ ...
1124
+
1125
+ @typing.overload
1126
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1127
+ ...
1128
+
1129
+ @typing.overload
1130
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1131
+ ...
1132
+
1133
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1134
+ """
1135
+ Specifies the resources needed when executing this step.
1136
+
1137
+ Use `@resources` to specify the resource requirements
1138
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1139
+
1140
+ You can choose the compute layer on the command line by executing e.g.
1141
+ ```
1142
+ python myflow.py run --with batch
1143
+ ```
1144
+ or
1145
+ ```
1146
+ python myflow.py run --with kubernetes
1147
+ ```
1148
+ which executes the flow on the desired system using the
1149
+ requirements specified in `@resources`.
1150
+
1151
+
1152
+ Parameters
1153
+ ----------
1154
+ cpu : int, default 1
1155
+ Number of CPUs required for this step.
1156
+ gpu : int, optional, default None
1157
+ Number of GPUs required for this step.
1158
+ disk : int, optional, default None
1159
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1160
+ memory : int, default 4096
1161
+ Memory size (in MB) required for this step.
1162
+ shared_memory : int, optional, default None
1163
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1164
+ This parameter maps to the `--shm-size` option in Docker.
1165
+ """
1166
+ ...
1167
+
1168
+ @typing.overload
1169
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1170
+ """
1171
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1172
+ It exists to make it easier for users to know that this decorator should only be used with
1173
+ a Neo Cloud like CoreWeave.
1174
+ """
1175
+ ...
1176
+
1177
+ @typing.overload
1178
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1179
+ ...
1180
+
1181
+ def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1182
+ """
1183
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1184
+ It exists to make it easier for users to know that this decorator should only be used with
1185
+ a Neo Cloud like CoreWeave.
1186
+ """
1187
+ ...
1188
+
1189
+ @typing.overload
1190
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1191
+ """
1192
+ Specifies secrets to be retrieved and injected as environment variables prior to
1193
+ the execution of a step.
1194
+
1195
+
1196
+ Parameters
1197
+ ----------
1198
+ sources : List[Union[str, Dict[str, Any]]], default: []
1199
+ List of secret specs, defining how the secrets are to be retrieved
1200
+ role : str, optional, default: None
1201
+ Role to use for fetching secrets
1202
+ """
1203
+ ...
1204
+
1205
+ @typing.overload
1206
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1207
+ ...
1208
+
1209
+ @typing.overload
1210
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1211
+ ...
1212
+
1213
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
1214
+ """
1215
+ Specifies secrets to be retrieved and injected as environment variables prior to
1216
+ the execution of a step.
1217
+
1218
+
1219
+ Parameters
1220
+ ----------
1221
+ sources : List[Union[str, Dict[str, Any]]], default: []
1222
+ List of secret specs, defining how the secrets are to be retrieved
1223
+ role : str, optional, default: None
1224
+ Role to use for fetching secrets
1225
+ """
1226
+ ...
1227
+
1228
+ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1229
+ """
1230
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
1231
+
1232
+ User code call
1233
+ --------------
1234
+ @vllm(
1235
+ model="...",
1236
+ ...
1237
+ )
1238
+
1239
+ Valid backend options
1240
+ ---------------------
1241
+ - 'local': Run as a separate process on the local task machine.
1242
+
1243
+ Valid model options
1244
+ -------------------
1245
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
1246
+
1247
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
1248
+ If you need multiple models, you must create multiple @vllm decorators.
1249
+
1250
+
1251
+ Parameters
1252
+ ----------
1253
+ model: str
1254
+ HuggingFace model identifier to be served by vLLM.
1255
+ backend: str
1256
+ Determines where and how to run the vLLM process.
1257
+ openai_api_server: bool
1258
+ Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
1259
+ Default is False (uses native engine).
1260
+ Set to True for backward compatibility with existing code.
1261
+ debug: bool
1262
+ Whether to turn on verbose debugging logs.
1263
+ card_refresh_interval: int
1264
+ Interval in seconds for refreshing the vLLM status card.
1265
+ Only used when openai_api_server=True.
1266
+ max_retries: int
1267
+ Maximum number of retries checking for vLLM server startup.
1268
+ Only used when openai_api_server=True.
1269
+ retry_alert_frequency: int
1270
+ Frequency of alert logs for vLLM server startup retries.
1271
+ Only used when openai_api_server=True.
1272
+ engine_args : dict
1273
+ Additional keyword arguments to pass to the vLLM engine.
1274
+ For example, `tensor_parallel_size=2`.
1275
+ """
1276
+ ...
1277
+
1278
+ @typing.overload
1279
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1280
+ """
1281
+ Specifies that the step will success under all circumstances.
1282
+
1283
+ The decorator will create an optional artifact, specified by `var`, which
1284
+ contains the exception raised. You can use it to detect the presence
1285
+ of errors, indicating that all happy-path artifacts produced by the step
1286
+ are missing.
1287
+
1288
+
1289
+ Parameters
1290
+ ----------
1291
+ var : str, optional, default None
1292
+ Name of the artifact in which to store the caught exception.
1293
+ If not specified, the exception is not stored.
1294
+ print_exception : bool, default True
1295
+ Determines whether or not the exception is printed to
1296
+ stdout when caught.
1297
+ """
1298
+ ...
1299
+
1300
+ @typing.overload
1301
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1302
+ ...
1303
+
1304
+ @typing.overload
1305
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1306
+ ...
1307
+
1308
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1309
+ """
1310
+ Specifies that the step will success under all circumstances.
1311
+
1312
+ The decorator will create an optional artifact, specified by `var`, which
1313
+ contains the exception raised. You can use it to detect the presence
1314
+ of errors, indicating that all happy-path artifacts produced by the step
1315
+ are missing.
1316
+
1317
+
1318
+ Parameters
1319
+ ----------
1320
+ var : str, optional, default None
1321
+ Name of the artifact in which to store the caught exception.
1322
+ If not specified, the exception is not stored.
1323
+ print_exception : bool, default True
1324
+ Determines whether or not the exception is printed to
1325
+ stdout when caught.
1326
+ """
1327
+ ...
1328
+
1329
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1330
+ """
1331
+ S3 Proxy decorator for routing S3 requests through a local proxy service.
1332
+
1333
+
1334
+ Parameters
1335
+ ----------
1336
+ integration_name : str, optional
1337
+ Name of the S3 proxy integration. If not specified, will use the only
1338
+ available S3 proxy integration in the namespace (fails if multiple exist).
1339
+ write_mode : str, optional
1340
+ The desired behavior during write operations to target (origin) S3 bucket.
1341
+ allowed options are:
1342
+ "origin-and-cache" -> write to both the target S3 bucket and local object
1343
+ storage
1344
+ "origin" -> only write to the target S3 bucket
1345
+ "cache" -> only write to the object storage service used for caching
1346
+ debug : bool, optional
1347
+ Enable debug logging for proxy operations.
1348
+ """
1349
+ ...
1350
+
1351
+ @typing.overload
1352
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1353
+ """
1354
+ Specifies the event(s) that this flow depends on.
1355
+
1356
+ ```
1357
+ @trigger(event='foo')
1358
+ ```
1359
+ or
1360
+ ```
1361
+ @trigger(events=['foo', 'bar'])
1362
+ ```
1363
+
1364
+ Additionally, you can specify the parameter mappings
1365
+ to map event payload to Metaflow parameters for the flow.
1366
+ ```
1367
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1368
+ ```
1369
+ or
1370
+ ```
1371
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1372
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1373
+ ```
1374
+
1375
+ 'parameters' can also be a list of strings and tuples like so:
1376
+ ```
1377
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1378
+ ```
1379
+ This is equivalent to:
1380
+ ```
1381
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1382
+ ```
1383
+
1384
+
1385
+ Parameters
1386
+ ----------
1387
+ event : Union[str, Dict[str, Any]], optional, default None
1388
+ Event dependency for this flow.
1389
+ events : List[Union[str, Dict[str, Any]]], default []
1390
+ Events dependency for this flow.
1391
+ options : Dict[str, Any], default {}
1392
+ Backend-specific configuration for tuning eventing behavior.
1393
+ """
1394
+ ...
1395
+
1396
+ @typing.overload
1397
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1398
+ ...
1399
+
1400
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1401
+ """
1402
+ Specifies the event(s) that this flow depends on.
1403
+
1404
+ ```
1405
+ @trigger(event='foo')
1406
+ ```
1407
+ or
1408
+ ```
1409
+ @trigger(events=['foo', 'bar'])
1410
+ ```
1411
+
1412
+ Additionally, you can specify the parameter mappings
1413
+ to map event payload to Metaflow parameters for the flow.
1414
+ ```
1415
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1416
+ ```
1417
+ or
1418
+ ```
1419
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1420
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1421
+ ```
1422
+
1423
+ 'parameters' can also be a list of strings and tuples like so:
1424
+ ```
1425
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1426
+ ```
1427
+ This is equivalent to:
1428
+ ```
1429
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1147
1430
  ```
1148
1431
 
1149
1432
 
1150
1433
  Parameters
1151
1434
  ----------
1152
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1153
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1154
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1155
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1156
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1157
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1158
-
1159
- temp_dir_root : str, default: None
1160
- The root directory under which `current.model.loaded` will store loaded models
1435
+ event : Union[str, Dict[str, Any]], optional, default None
1436
+ Event dependency for this flow.
1437
+ events : List[Union[str, Dict[str, Any]]], default []
1438
+ Events dependency for this flow.
1439
+ options : Dict[str, Any], default {}
1440
+ Backend-specific configuration for tuning eventing behavior.
1161
1441
  """
1162
1442
  ...
1163
1443
 
1164
- def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1444
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1165
1445
  """
1166
- This decorator is used to run vllm APIs as Metaflow task sidecars.
1167
-
1168
- User code call
1169
- --------------
1170
- @vllm(
1171
- model="...",
1172
- ...
1173
- )
1174
-
1175
- Valid backend options
1176
- ---------------------
1177
- - 'local': Run as a separate process on the local task machine.
1178
-
1179
- Valid model options
1180
- -------------------
1181
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
1182
-
1183
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
1184
- If you need multiple models, you must create multiple @vllm decorators.
1446
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1447
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1448
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1449
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1450
+ starts only after all sensors finish.
1185
1451
 
1186
1452
 
1187
1453
  Parameters
1188
1454
  ----------
1189
- model: str
1190
- HuggingFace model identifier to be served by vLLM.
1191
- backend: str
1192
- Determines where and how to run the vLLM process.
1193
- openai_api_server: bool
1194
- Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
1195
- Default is False (uses native engine).
1196
- Set to True for backward compatibility with existing code.
1197
- debug: bool
1198
- Whether to turn on verbose debugging logs.
1199
- card_refresh_interval: int
1200
- Interval in seconds for refreshing the vLLM status card.
1201
- Only used when openai_api_server=True.
1202
- max_retries: int
1203
- Maximum number of retries checking for vLLM server startup.
1204
- Only used when openai_api_server=True.
1205
- retry_alert_frequency: int
1206
- Frequency of alert logs for vLLM server startup retries.
1207
- Only used when openai_api_server=True.
1208
- engine_args : dict
1209
- Additional keyword arguments to pass to the vLLM engine.
1210
- For example, `tensor_parallel_size=2`.
1455
+ timeout : int
1456
+ Time, in seconds before the task times out and fails. (Default: 3600)
1457
+ poke_interval : int
1458
+ Time in seconds that the job should wait in between each try. (Default: 60)
1459
+ mode : str
1460
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1461
+ exponential_backoff : bool
1462
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1463
+ pool : str
1464
+ the slot pool this task should run in,
1465
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1466
+ soft_fail : bool
1467
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1468
+ name : str
1469
+ Name of the sensor on Airflow
1470
+ description : str
1471
+ Description of sensor in the Airflow UI
1472
+ bucket_key : Union[str, List[str]]
1473
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1474
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1475
+ bucket_name : str
1476
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1477
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1478
+ wildcard_match : bool
1479
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1480
+ aws_conn_id : str
1481
+ a reference to the s3 connection on Airflow. (Default: None)
1482
+ verify : bool
1483
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1211
1484
  """
1212
1485
  ...
1213
1486
 
1214
1487
  @typing.overload
1215
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1488
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1216
1489
  """
1217
- Specifies the Conda environment for the step.
1490
+ Specifies the Conda environment for all steps of the flow.
1218
1491
 
1219
- Information in this decorator will augment any
1220
- attributes set in the `@conda_base` flow-level decorator. Hence,
1221
- you can use `@conda_base` to set packages required by all
1222
- steps and use `@conda` to specify step-specific overrides.
1492
+ Use `@conda_base` to set common libraries required by all
1493
+ steps and use `@conda` to specify step-specific additions.
1223
1494
 
1224
1495
 
1225
1496
  Parameters
1226
1497
  ----------
1227
1498
  packages : Dict[str, str], default {}
1228
- Packages to use for this step. The key is the name of the package
1499
+ Packages to use for this flow. The key is the name of the package
1229
1500
  and the value is the version to use.
1230
1501
  libraries : Dict[str, str], default {}
1231
1502
  Supported for backward compatibility. When used with packages, packages will take precedence.
@@ -1233,32 +1504,26 @@ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, s
1233
1504
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1234
1505
  that the version used will correspond to the version of the Python interpreter used to start the run.
1235
1506
  disabled : bool, default False
1236
- If set to True, disables @conda.
1507
+ If set to True, disables Conda.
1237
1508
  """
1238
1509
  ...
1239
1510
 
1240
1511
  @typing.overload
1241
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1242
- ...
1243
-
1244
- @typing.overload
1245
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1512
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1246
1513
  ...
1247
1514
 
1248
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1515
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1249
1516
  """
1250
- Specifies the Conda environment for the step.
1517
+ Specifies the Conda environment for all steps of the flow.
1251
1518
 
1252
- Information in this decorator will augment any
1253
- attributes set in the `@conda_base` flow-level decorator. Hence,
1254
- you can use `@conda_base` to set packages required by all
1255
- steps and use `@conda` to specify step-specific overrides.
1519
+ Use `@conda_base` to set common libraries required by all
1520
+ steps and use `@conda` to specify step-specific additions.
1256
1521
 
1257
1522
 
1258
1523
  Parameters
1259
1524
  ----------
1260
1525
  packages : Dict[str, str], default {}
1261
- Packages to use for this step. The key is the name of the package
1526
+ Packages to use for this flow. The key is the name of the package
1262
1527
  and the value is the version to use.
1263
1528
  libraries : Dict[str, str], default {}
1264
1529
  Supported for backward compatibility. When used with packages, packages will take precedence.
@@ -1266,90 +1531,39 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1266
1531
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1267
1532
  that the version used will correspond to the version of the Python interpreter used to start the run.
1268
1533
  disabled : bool, default False
1269
- If set to True, disables @conda.
1270
- """
1271
- ...
1272
-
1273
- @typing.overload
1274
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1275
- """
1276
- Decorator prototype for all step decorators. This function gets specialized
1277
- and imported for all decorators types by _import_plugin_decorators().
1278
- """
1279
- ...
1280
-
1281
- @typing.overload
1282
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1283
- ...
1284
-
1285
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1286
- """
1287
- Decorator prototype for all step decorators. This function gets specialized
1288
- and imported for all decorators types by _import_plugin_decorators().
1534
+ If set to True, disables Conda.
1289
1535
  """
1290
1536
  ...
1291
1537
 
1292
1538
  @typing.overload
1293
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1539
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1294
1540
  """
1295
- Specifies a timeout for your step.
1296
-
1297
- This decorator is useful if this step may hang indefinitely.
1298
-
1299
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1300
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1301
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1302
-
1303
- Note that all the values specified in parameters are added together so if you specify
1304
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1541
+ Specifies the times when the flow should be run when running on a
1542
+ production scheduler.
1305
1543
 
1306
1544
 
1307
1545
  Parameters
1308
1546
  ----------
1309
- seconds : int, default 0
1310
- Number of seconds to wait prior to timing out.
1311
- minutes : int, default 0
1312
- Number of minutes to wait prior to timing out.
1313
- hours : int, default 0
1314
- Number of hours to wait prior to timing out.
1547
+ hourly : bool, default False
1548
+ Run the workflow hourly.
1549
+ daily : bool, default True
1550
+ Run the workflow daily.
1551
+ weekly : bool, default False
1552
+ Run the workflow weekly.
1553
+ cron : str, optional, default None
1554
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1555
+ specified by this expression.
1556
+ timezone : str, optional, default None
1557
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1558
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1315
1559
  """
1316
1560
  ...
1317
1561
 
1318
1562
  @typing.overload
1319
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1320
- ...
1321
-
1322
- @typing.overload
1323
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1324
- ...
1325
-
1326
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1327
- """
1328
- Specifies a timeout for your step.
1329
-
1330
- This decorator is useful if this step may hang indefinitely.
1331
-
1332
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1333
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1334
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1335
-
1336
- Note that all the values specified in parameters are added together so if you specify
1337
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1338
-
1339
-
1340
- Parameters
1341
- ----------
1342
- seconds : int, default 0
1343
- Number of seconds to wait prior to timing out.
1344
- minutes : int, default 0
1345
- Number of minutes to wait prior to timing out.
1346
- hours : int, default 0
1347
- Number of hours to wait prior to timing out.
1348
- """
1563
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1349
1564
  ...
1350
1565
 
1351
- @typing.overload
1352
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1566
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1353
1567
  """
1354
1568
  Specifies the times when the flow should be run when running on a
1355
1569
  production scheduler.
@@ -1373,29 +1587,43 @@ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False,
1373
1587
  ...
1374
1588
 
1375
1589
  @typing.overload
1376
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1590
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1591
+ """
1592
+ Specifies the PyPI packages for all steps of the flow.
1593
+
1594
+ Use `@pypi_base` to set common packages required by all
1595
+ steps and use `@pypi` to specify step-specific overrides.
1596
+
1597
+ Parameters
1598
+ ----------
1599
+ packages : Dict[str, str], default: {}
1600
+ Packages to use for this flow. The key is the name of the package
1601
+ and the value is the version to use.
1602
+ python : str, optional, default: None
1603
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1604
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1605
+ """
1606
+ ...
1607
+
1608
+ @typing.overload
1609
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1377
1610
  ...
1378
1611
 
1379
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1612
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1380
1613
  """
1381
- Specifies the times when the flow should be run when running on a
1382
- production scheduler.
1614
+ Specifies the PyPI packages for all steps of the flow.
1383
1615
 
1616
+ Use `@pypi_base` to set common packages required by all
1617
+ steps and use `@pypi` to specify step-specific overrides.
1384
1618
 
1385
1619
  Parameters
1386
1620
  ----------
1387
- hourly : bool, default False
1388
- Run the workflow hourly.
1389
- daily : bool, default True
1390
- Run the workflow daily.
1391
- weekly : bool, default False
1392
- Run the workflow weekly.
1393
- cron : str, optional, default None
1394
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1395
- specified by this expression.
1396
- timezone : str, optional, default None
1397
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1398
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1621
+ packages : Dict[str, str], default: {}
1622
+ Packages to use for this flow. The key is the name of the package
1623
+ and the value is the version to use.
1624
+ python : str, optional, default: None
1625
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1626
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1399
1627
  """
1400
1628
  ...
1401
1629
 
@@ -1692,233 +1920,5 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1692
1920
  """
1693
1921
  ...
1694
1922
 
1695
- @typing.overload
1696
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1697
- """
1698
- Specifies the event(s) that this flow depends on.
1699
-
1700
- ```
1701
- @trigger(event='foo')
1702
- ```
1703
- or
1704
- ```
1705
- @trigger(events=['foo', 'bar'])
1706
- ```
1707
-
1708
- Additionally, you can specify the parameter mappings
1709
- to map event payload to Metaflow parameters for the flow.
1710
- ```
1711
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1712
- ```
1713
- or
1714
- ```
1715
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1716
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1717
- ```
1718
-
1719
- 'parameters' can also be a list of strings and tuples like so:
1720
- ```
1721
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1722
- ```
1723
- This is equivalent to:
1724
- ```
1725
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1726
- ```
1727
-
1728
-
1729
- Parameters
1730
- ----------
1731
- event : Union[str, Dict[str, Any]], optional, default None
1732
- Event dependency for this flow.
1733
- events : List[Union[str, Dict[str, Any]]], default []
1734
- Events dependency for this flow.
1735
- options : Dict[str, Any], default {}
1736
- Backend-specific configuration for tuning eventing behavior.
1737
- """
1738
- ...
1739
-
1740
- @typing.overload
1741
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1742
- ...
1743
-
1744
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1745
- """
1746
- Specifies the event(s) that this flow depends on.
1747
-
1748
- ```
1749
- @trigger(event='foo')
1750
- ```
1751
- or
1752
- ```
1753
- @trigger(events=['foo', 'bar'])
1754
- ```
1755
-
1756
- Additionally, you can specify the parameter mappings
1757
- to map event payload to Metaflow parameters for the flow.
1758
- ```
1759
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1760
- ```
1761
- or
1762
- ```
1763
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1764
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1765
- ```
1766
-
1767
- 'parameters' can also be a list of strings and tuples like so:
1768
- ```
1769
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1770
- ```
1771
- This is equivalent to:
1772
- ```
1773
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1774
- ```
1775
-
1776
-
1777
- Parameters
1778
- ----------
1779
- event : Union[str, Dict[str, Any]], optional, default None
1780
- Event dependency for this flow.
1781
- events : List[Union[str, Dict[str, Any]]], default []
1782
- Events dependency for this flow.
1783
- options : Dict[str, Any], default {}
1784
- Backend-specific configuration for tuning eventing behavior.
1785
- """
1786
- ...
1787
-
1788
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1789
- """
1790
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1791
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1792
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1793
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1794
- starts only after all sensors finish.
1795
-
1796
-
1797
- Parameters
1798
- ----------
1799
- timeout : int
1800
- Time, in seconds before the task times out and fails. (Default: 3600)
1801
- poke_interval : int
1802
- Time in seconds that the job should wait in between each try. (Default: 60)
1803
- mode : str
1804
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1805
- exponential_backoff : bool
1806
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1807
- pool : str
1808
- the slot pool this task should run in,
1809
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1810
- soft_fail : bool
1811
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1812
- name : str
1813
- Name of the sensor on Airflow
1814
- description : str
1815
- Description of sensor in the Airflow UI
1816
- bucket_key : Union[str, List[str]]
1817
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1818
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1819
- bucket_name : str
1820
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1821
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1822
- wildcard_match : bool
1823
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1824
- aws_conn_id : str
1825
- a reference to the s3 connection on Airflow. (Default: None)
1826
- verify : bool
1827
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1828
- """
1829
- ...
1830
-
1831
- @typing.overload
1832
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1833
- """
1834
- Specifies the Conda environment for all steps of the flow.
1835
-
1836
- Use `@conda_base` to set common libraries required by all
1837
- steps and use `@conda` to specify step-specific additions.
1838
-
1839
-
1840
- Parameters
1841
- ----------
1842
- packages : Dict[str, str], default {}
1843
- Packages to use for this flow. The key is the name of the package
1844
- and the value is the version to use.
1845
- libraries : Dict[str, str], default {}
1846
- Supported for backward compatibility. When used with packages, packages will take precedence.
1847
- python : str, optional, default None
1848
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1849
- that the version used will correspond to the version of the Python interpreter used to start the run.
1850
- disabled : bool, default False
1851
- If set to True, disables Conda.
1852
- """
1853
- ...
1854
-
1855
- @typing.overload
1856
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1857
- ...
1858
-
1859
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1860
- """
1861
- Specifies the Conda environment for all steps of the flow.
1862
-
1863
- Use `@conda_base` to set common libraries required by all
1864
- steps and use `@conda` to specify step-specific additions.
1865
-
1866
-
1867
- Parameters
1868
- ----------
1869
- packages : Dict[str, str], default {}
1870
- Packages to use for this flow. The key is the name of the package
1871
- and the value is the version to use.
1872
- libraries : Dict[str, str], default {}
1873
- Supported for backward compatibility. When used with packages, packages will take precedence.
1874
- python : str, optional, default None
1875
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1876
- that the version used will correspond to the version of the Python interpreter used to start the run.
1877
- disabled : bool, default False
1878
- If set to True, disables Conda.
1879
- """
1880
- ...
1881
-
1882
- @typing.overload
1883
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1884
- """
1885
- Specifies the PyPI packages for all steps of the flow.
1886
-
1887
- Use `@pypi_base` to set common packages required by all
1888
- steps and use `@pypi` to specify step-specific overrides.
1889
-
1890
- Parameters
1891
- ----------
1892
- packages : Dict[str, str], default: {}
1893
- Packages to use for this flow. The key is the name of the package
1894
- and the value is the version to use.
1895
- python : str, optional, default: None
1896
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1897
- that the version used will correspond to the version of the Python interpreter used to start the run.
1898
- """
1899
- ...
1900
-
1901
- @typing.overload
1902
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1903
- ...
1904
-
1905
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1906
- """
1907
- Specifies the PyPI packages for all steps of the flow.
1908
-
1909
- Use `@pypi_base` to set common packages required by all
1910
- steps and use `@pypi` to specify step-specific overrides.
1911
-
1912
- Parameters
1913
- ----------
1914
- packages : Dict[str, str], default: {}
1915
- Packages to use for this flow. The key is the name of the package
1916
- and the value is the version to use.
1917
- python : str, optional, default: None
1918
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1919
- that the version used will correspond to the version of the Python interpreter used to start the run.
1920
- """
1921
- ...
1922
-
1923
1923
  pkg_name: str
1924
1924