ob-metaflow-stubs 6.0.8.1__py2.py3-none-any.whl → 6.0.8.3__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow-stubs might be problematic. Click here for more details.

Files changed (262) hide show
  1. metaflow-stubs/__init__.pyi +930 -930
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +4 -4
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +3 -3
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/meta_files.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +66 -66
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +1 -1
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +1 -1
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +1 -1
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +1 -1
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +1 -1
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +3 -3
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +1 -1
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +1 -1
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +1 -1
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +1 -1
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +3 -3
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +1 -1
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +2 -2
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +3 -3
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +4 -4
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +1 -1
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +1 -1
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +1 -1
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +1 -1
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +3 -3
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +3 -3
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +4 -4
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +1 -1
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  110. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  111. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  112. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  113. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  114. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +1 -1
  116. metaflow-stubs/multicore_utils.pyi +1 -1
  117. metaflow-stubs/ob_internal.pyi +1 -1
  118. metaflow-stubs/packaging_sys/__init__.pyi +5 -5
  119. metaflow-stubs/packaging_sys/backend.pyi +2 -2
  120. metaflow-stubs/packaging_sys/distribution_support.pyi +3 -3
  121. metaflow-stubs/packaging_sys/tar_backend.pyi +4 -4
  122. metaflow-stubs/packaging_sys/utils.pyi +1 -1
  123. metaflow-stubs/packaging_sys/v1.pyi +3 -3
  124. metaflow-stubs/parameters.pyi +4 -4
  125. metaflow-stubs/plugins/__init__.pyi +12 -12
  126. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  127. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  128. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  129. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  130. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  131. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  132. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  133. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  134. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  135. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  136. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  137. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  138. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  139. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  140. metaflow-stubs/plugins/argo/exit_hooks.pyi +2 -2
  141. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  142. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  143. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  144. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  145. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  146. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  147. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  148. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  149. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  150. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  151. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  152. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  153. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  154. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  155. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  157. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  158. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  159. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  160. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  161. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  162. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  163. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  164. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  165. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  166. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  167. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  168. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  169. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  170. metaflow-stubs/plugins/cards/card_modules/basic.pyi +1 -1
  171. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  172. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  173. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  174. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  175. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  176. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  177. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  178. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  179. metaflow-stubs/plugins/catch_decorator.pyi +1 -1
  180. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  181. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  182. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  183. metaflow-stubs/plugins/datatools/s3/s3.pyi +2 -2
  184. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  185. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  186. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  187. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  188. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  189. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  190. metaflow-stubs/plugins/exit_hook/__init__.pyi +1 -1
  191. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +1 -1
  192. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  193. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  194. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  195. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  196. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  197. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  198. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  199. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  200. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  201. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  202. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  203. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  204. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  205. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  206. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  207. metaflow-stubs/plugins/ollama/__init__.pyi +1 -1
  208. metaflow-stubs/plugins/optuna/__init__.pyi +1 -1
  209. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  210. metaflow-stubs/plugins/perimeters.pyi +1 -1
  211. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  212. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  213. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  214. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  215. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  216. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  217. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  218. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  219. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  220. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  221. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  222. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
  223. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  224. metaflow-stubs/plugins/secrets/secrets_func.pyi +1 -1
  225. metaflow-stubs/plugins/secrets/secrets_spec.pyi +1 -1
  226. metaflow-stubs/plugins/secrets/utils.pyi +1 -1
  227. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  228. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  229. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  230. metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
  231. metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
  232. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  233. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  234. metaflow-stubs/profilers/__init__.pyi +1 -1
  235. metaflow-stubs/pylint_wrapper.pyi +1 -1
  236. metaflow-stubs/runner/__init__.pyi +1 -1
  237. metaflow-stubs/runner/deployer.pyi +4 -4
  238. metaflow-stubs/runner/deployer_impl.pyi +1 -1
  239. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  240. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  241. metaflow-stubs/runner/nbrun.pyi +1 -1
  242. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  243. metaflow-stubs/runner/utils.pyi +3 -3
  244. metaflow-stubs/system/__init__.pyi +1 -1
  245. metaflow-stubs/system/system_logger.pyi +2 -2
  246. metaflow-stubs/system/system_monitor.pyi +1 -1
  247. metaflow-stubs/tagging_util.pyi +1 -1
  248. metaflow-stubs/tuple_util.pyi +1 -1
  249. metaflow-stubs/user_configs/__init__.pyi +1 -1
  250. metaflow-stubs/user_configs/config_options.pyi +3 -3
  251. metaflow-stubs/user_configs/config_parameters.pyi +4 -4
  252. metaflow-stubs/user_decorators/__init__.pyi +1 -1
  253. metaflow-stubs/user_decorators/common.pyi +1 -1
  254. metaflow-stubs/user_decorators/mutable_flow.pyi +4 -4
  255. metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
  256. metaflow-stubs/user_decorators/user_flow_decorator.pyi +3 -3
  257. metaflow-stubs/user_decorators/user_step_decorator.pyi +4 -4
  258. {ob_metaflow_stubs-6.0.8.1.dist-info → ob_metaflow_stubs-6.0.8.3.dist-info}/METADATA +1 -1
  259. ob_metaflow_stubs-6.0.8.3.dist-info/RECORD +262 -0
  260. ob_metaflow_stubs-6.0.8.1.dist-info/RECORD +0 -262
  261. {ob_metaflow_stubs-6.0.8.1.dist-info → ob_metaflow_stubs-6.0.8.3.dist-info}/WHEEL +0 -0
  262. {ob_metaflow_stubs-6.0.8.1.dist-info → ob_metaflow_stubs-6.0.8.3.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.17.1.0+obcheckpoint(0.2.4);ob(v1) #
4
- # Generated on 2025-08-21T23:31:59.916830 #
4
+ # Generated on 2025-08-25T21:23:22.454608 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -40,17 +40,17 @@ from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
42
  from . import cards as cards
43
- from . import tuple_util as tuple_util
44
- from . import metaflow_git as metaflow_git
45
43
  from . import events as events
44
+ from . import metaflow_git as metaflow_git
45
+ from . import tuple_util as tuple_util
46
46
  from . import runner as runner
47
47
  from . import plugins as plugins
48
48
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
49
49
  from . import includefile as includefile
50
50
  from .includefile import IncludeFile as IncludeFile
51
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
52
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
53
51
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
52
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
53
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
54
54
  from . import client as client
55
55
  from .client.core import namespace as namespace
56
56
  from .client.core import get_namespace as get_namespace
@@ -167,225 +167,407 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
167
167
  """
168
168
  ...
169
169
 
170
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
170
+ @typing.overload
171
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
171
172
  """
172
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
173
+ Specifies a timeout for your step.
173
174
 
174
- User code call
175
- --------------
176
- @ollama(
177
- models=[...],
178
- ...
179
- )
175
+ This decorator is useful if this step may hang indefinitely.
180
176
 
181
- Valid backend options
182
- ---------------------
183
- - 'local': Run as a separate process on the local task machine.
184
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
185
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
177
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
178
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
179
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
186
180
 
187
- Valid model options
188
- -------------------
189
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
181
+ Note that all the values specified in parameters are added together so if you specify
182
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
190
183
 
191
184
 
192
185
  Parameters
193
186
  ----------
194
- models: list[str]
195
- List of Ollama containers running models in sidecars.
196
- backend: str
197
- Determines where and how to run the Ollama process.
198
- force_pull: bool
199
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
200
- cache_update_policy: str
201
- Cache update policy: "auto", "force", or "never".
202
- force_cache_update: bool
203
- Simple override for "force" cache update policy.
204
- debug: bool
205
- Whether to turn on verbose debugging logs.
206
- circuit_breaker_config: dict
207
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
208
- timeout_config: dict
209
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
187
+ seconds : int, default 0
188
+ Number of seconds to wait prior to timing out.
189
+ minutes : int, default 0
190
+ Number of minutes to wait prior to timing out.
191
+ hours : int, default 0
192
+ Number of hours to wait prior to timing out.
210
193
  """
211
194
  ...
212
195
 
213
196
  @typing.overload
214
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
197
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
198
+ ...
199
+
200
+ @typing.overload
201
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
202
+ ...
203
+
204
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
215
205
  """
216
- Specifies the number of times the task corresponding
217
- to a step needs to be retried.
206
+ Specifies a timeout for your step.
218
207
 
219
- This decorator is useful for handling transient errors, such as networking issues.
220
- If your task contains operations that can't be retried safely, e.g. database updates,
221
- it is advisable to annotate it with `@retry(times=0)`.
208
+ This decorator is useful if this step may hang indefinitely.
222
209
 
223
- This can be used in conjunction with the `@catch` decorator. The `@catch`
224
- decorator will execute a no-op task after all retries have been exhausted,
225
- ensuring that the flow execution can continue.
210
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
211
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
212
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
213
+
214
+ Note that all the values specified in parameters are added together so if you specify
215
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
226
216
 
227
217
 
228
218
  Parameters
229
219
  ----------
230
- times : int, default 3
231
- Number of times to retry this task.
232
- minutes_between_retries : int, default 2
233
- Number of minutes between retries.
220
+ seconds : int, default 0
221
+ Number of seconds to wait prior to timing out.
222
+ minutes : int, default 0
223
+ Number of minutes to wait prior to timing out.
224
+ hours : int, default 0
225
+ Number of hours to wait prior to timing out.
234
226
  """
235
227
  ...
236
228
 
237
229
  @typing.overload
238
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
230
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
231
+ """
232
+ Specifies secrets to be retrieved and injected as environment variables prior to
233
+ the execution of a step.
234
+
235
+
236
+ Parameters
237
+ ----------
238
+ sources : List[Union[str, Dict[str, Any]]], default: []
239
+ List of secret specs, defining how the secrets are to be retrieved
240
+ role : str, optional, default: None
241
+ Role to use for fetching secrets
242
+ """
239
243
  ...
240
244
 
241
245
  @typing.overload
242
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
246
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
243
247
  ...
244
248
 
245
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
249
+ @typing.overload
250
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
251
+ ...
252
+
253
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
246
254
  """
247
- Specifies the number of times the task corresponding
248
- to a step needs to be retried.
249
-
250
- This decorator is useful for handling transient errors, such as networking issues.
251
- If your task contains operations that can't be retried safely, e.g. database updates,
252
- it is advisable to annotate it with `@retry(times=0)`.
253
-
254
- This can be used in conjunction with the `@catch` decorator. The `@catch`
255
- decorator will execute a no-op task after all retries have been exhausted,
256
- ensuring that the flow execution can continue.
255
+ Specifies secrets to be retrieved and injected as environment variables prior to
256
+ the execution of a step.
257
257
 
258
258
 
259
259
  Parameters
260
260
  ----------
261
- times : int, default 3
262
- Number of times to retry this task.
263
- minutes_between_retries : int, default 2
264
- Number of minutes between retries.
261
+ sources : List[Union[str, Dict[str, Any]]], default: []
262
+ List of secret specs, defining how the secrets are to be retrieved
263
+ role : str, optional, default: None
264
+ Role to use for fetching secrets
265
265
  """
266
266
  ...
267
267
 
268
268
  @typing.overload
269
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
269
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
270
270
  """
271
- Specifies that the step will success under all circumstances.
271
+ Creates a human-readable report, a Metaflow Card, after this step completes.
272
272
 
273
- The decorator will create an optional artifact, specified by `var`, which
274
- contains the exception raised. You can use it to detect the presence
275
- of errors, indicating that all happy-path artifacts produced by the step
276
- are missing.
273
+ Note that you may add multiple `@card` decorators in a step with different parameters.
277
274
 
278
275
 
279
276
  Parameters
280
277
  ----------
281
- var : str, optional, default None
282
- Name of the artifact in which to store the caught exception.
283
- If not specified, the exception is not stored.
284
- print_exception : bool, default True
285
- Determines whether or not the exception is printed to
286
- stdout when caught.
278
+ type : str, default 'default'
279
+ Card type.
280
+ id : str, optional, default None
281
+ If multiple cards are present, use this id to identify this card.
282
+ options : Dict[str, Any], default {}
283
+ Options passed to the card. The contents depend on the card type.
284
+ timeout : int, default 45
285
+ Interrupt reporting if it takes more than this many seconds.
287
286
  """
288
287
  ...
289
288
 
290
289
  @typing.overload
291
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
290
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
292
291
  ...
293
292
 
294
293
  @typing.overload
295
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
294
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
296
295
  ...
297
296
 
298
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
297
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
299
298
  """
300
- Specifies that the step will success under all circumstances.
299
+ Creates a human-readable report, a Metaflow Card, after this step completes.
301
300
 
302
- The decorator will create an optional artifact, specified by `var`, which
303
- contains the exception raised. You can use it to detect the presence
304
- of errors, indicating that all happy-path artifacts produced by the step
305
- are missing.
301
+ Note that you may add multiple `@card` decorators in a step with different parameters.
306
302
 
307
303
 
308
304
  Parameters
309
305
  ----------
310
- var : str, optional, default None
311
- Name of the artifact in which to store the caught exception.
312
- If not specified, the exception is not stored.
313
- print_exception : bool, default True
314
- Determines whether or not the exception is printed to
315
- stdout when caught.
306
+ type : str, default 'default'
307
+ Card type.
308
+ id : str, optional, default None
309
+ If multiple cards are present, use this id to identify this card.
310
+ options : Dict[str, Any], default {}
311
+ Options passed to the card. The contents depend on the card type.
312
+ timeout : int, default 45
313
+ Interrupt reporting if it takes more than this many seconds.
316
314
  """
317
315
  ...
318
316
 
319
317
  @typing.overload
320
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
318
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
321
319
  """
322
- Enables loading / saving of models within a step.
323
-
324
- > Examples
325
- - Saving Models
326
- ```python
327
- @model
328
- @step
329
- def train(self):
330
- # current.model.save returns a dictionary reference to the model saved
331
- self.my_model = current.model.save(
332
- path_to_my_model,
333
- label="my_model",
334
- metadata={
335
- "epochs": 10,
336
- "batch-size": 32,
337
- "learning-rate": 0.001,
338
- }
339
- )
340
- self.next(self.test)
341
-
342
- @model(load="my_model")
343
- @step
344
- def test(self):
345
- # `current.model.loaded` returns a dictionary of the loaded models
346
- # where the key is the name of the artifact and the value is the path to the model
347
- print(os.listdir(current.model.loaded["my_model"]))
348
- self.next(self.end)
349
- ```
320
+ Specifies the Conda environment for the step.
350
321
 
351
- - Loading models
352
- ```python
353
- @step
354
- def train(self):
355
- # current.model.load returns the path to the model loaded
356
- checkpoint_path = current.model.load(
357
- self.checkpoint_key,
358
- )
359
- model_path = current.model.load(
360
- self.model,
361
- )
362
- self.next(self.test)
363
- ```
322
+ Information in this decorator will augment any
323
+ attributes set in the `@conda_base` flow-level decorator. Hence,
324
+ you can use `@conda_base` to set packages required by all
325
+ steps and use `@conda` to specify step-specific overrides.
364
326
 
365
327
 
366
328
  Parameters
367
329
  ----------
368
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
369
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
370
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
371
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
372
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
373
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
374
-
375
- temp_dir_root : str, default: None
376
- The root directory under which `current.model.loaded` will store loaded models
330
+ packages : Dict[str, str], default {}
331
+ Packages to use for this step. The key is the name of the package
332
+ and the value is the version to use.
333
+ libraries : Dict[str, str], default {}
334
+ Supported for backward compatibility. When used with packages, packages will take precedence.
335
+ python : str, optional, default None
336
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
337
+ that the version used will correspond to the version of the Python interpreter used to start the run.
338
+ disabled : bool, default False
339
+ If set to True, disables @conda.
377
340
  """
378
341
  ...
379
342
 
380
343
  @typing.overload
381
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
344
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
382
345
  ...
383
346
 
384
347
  @typing.overload
385
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
348
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
386
349
  ...
387
350
 
388
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
351
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
352
+ """
353
+ Specifies the Conda environment for the step.
354
+
355
+ Information in this decorator will augment any
356
+ attributes set in the `@conda_base` flow-level decorator. Hence,
357
+ you can use `@conda_base` to set packages required by all
358
+ steps and use `@conda` to specify step-specific overrides.
359
+
360
+
361
+ Parameters
362
+ ----------
363
+ packages : Dict[str, str], default {}
364
+ Packages to use for this step. The key is the name of the package
365
+ and the value is the version to use.
366
+ libraries : Dict[str, str], default {}
367
+ Supported for backward compatibility. When used with packages, packages will take precedence.
368
+ python : str, optional, default None
369
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
370
+ that the version used will correspond to the version of the Python interpreter used to start the run.
371
+ disabled : bool, default False
372
+ If set to True, disables @conda.
373
+ """
374
+ ...
375
+
376
+ @typing.overload
377
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
378
+ """
379
+ Internal decorator to support Fast bakery
380
+ """
381
+ ...
382
+
383
+ @typing.overload
384
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
385
+ ...
386
+
387
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
388
+ """
389
+ Internal decorator to support Fast bakery
390
+ """
391
+ ...
392
+
393
+ @typing.overload
394
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
395
+ """
396
+ Specifies the number of times the task corresponding
397
+ to a step needs to be retried.
398
+
399
+ This decorator is useful for handling transient errors, such as networking issues.
400
+ If your task contains operations that can't be retried safely, e.g. database updates,
401
+ it is advisable to annotate it with `@retry(times=0)`.
402
+
403
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
404
+ decorator will execute a no-op task after all retries have been exhausted,
405
+ ensuring that the flow execution can continue.
406
+
407
+
408
+ Parameters
409
+ ----------
410
+ times : int, default 3
411
+ Number of times to retry this task.
412
+ minutes_between_retries : int, default 2
413
+ Number of minutes between retries.
414
+ """
415
+ ...
416
+
417
+ @typing.overload
418
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
419
+ ...
420
+
421
+ @typing.overload
422
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
423
+ ...
424
+
425
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
426
+ """
427
+ Specifies the number of times the task corresponding
428
+ to a step needs to be retried.
429
+
430
+ This decorator is useful for handling transient errors, such as networking issues.
431
+ If your task contains operations that can't be retried safely, e.g. database updates,
432
+ it is advisable to annotate it with `@retry(times=0)`.
433
+
434
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
435
+ decorator will execute a no-op task after all retries have been exhausted,
436
+ ensuring that the flow execution can continue.
437
+
438
+
439
+ Parameters
440
+ ----------
441
+ times : int, default 3
442
+ Number of times to retry this task.
443
+ minutes_between_retries : int, default 2
444
+ Number of minutes between retries.
445
+ """
446
+ ...
447
+
448
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
449
+ """
450
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
451
+
452
+ > Examples
453
+
454
+ **Usage: creating references of models from huggingface that may be loaded in downstream steps**
455
+ ```python
456
+ @huggingface_hub
457
+ @step
458
+ def pull_model_from_huggingface(self):
459
+ # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
460
+ # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
461
+ # with the same `repo_id` in the backend storage, it will not download the model again. The return
462
+ # value of the function is a reference to the model in the backend storage.
463
+ # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
464
+
465
+ self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
466
+ self.llama_model = current.huggingface_hub.snapshot_download(
467
+ repo_id=self.model_id,
468
+ allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
469
+ )
470
+ self.next(self.train)
471
+ ```
472
+
473
+ **Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
474
+ ```python
475
+ @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
476
+ @step
477
+ def pull_model_from_huggingface(self):
478
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
479
+ ```
480
+
481
+ ```python
482
+ @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
483
+ @step
484
+ def finetune_model(self):
485
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
486
+ # path_to_model will be /my-directory
487
+ ```
488
+
489
+ ```python
490
+ # Takes all the arguments passed to `snapshot_download`
491
+ # except for `local_dir`
492
+ @huggingface_hub(load=[
493
+ {
494
+ "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
495
+ },
496
+ {
497
+ "repo_id": "myorg/mistral-lora",
498
+ "repo_type": "model",
499
+ },
500
+ ])
501
+ @step
502
+ def finetune_model(self):
503
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
504
+ # path_to_model will be /my-directory
505
+ ```
506
+
507
+
508
+ Parameters
509
+ ----------
510
+ temp_dir_root : str, optional
511
+ The root directory that will hold the temporary directory where objects will be downloaded.
512
+
513
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
514
+ The list of repos (models/datasets) to load.
515
+
516
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
517
+
518
+ - If repo (model/dataset) is not found in the datastore:
519
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
520
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
521
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
522
+
523
+ - If repo is found in the datastore:
524
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
525
+ """
526
+ ...
527
+
528
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
529
+ """
530
+ S3 Proxy decorator for routing S3 requests through a local proxy service.
531
+
532
+
533
+ Parameters
534
+ ----------
535
+ integration_name : str, optional
536
+ Name of the S3 proxy integration. If not specified, will use the only
537
+ available S3 proxy integration in the namespace (fails if multiple exist).
538
+ write_mode : str, optional
539
+ The desired behavior during write operations to target (origin) S3 bucket.
540
+ allowed options are:
541
+ "origin-and-cache" -> write to both the target S3 bucket and local object
542
+ storage
543
+ "origin" -> only write to the target S3 bucket
544
+ "cache" -> only write to the object storage service used for caching
545
+ debug : bool, optional
546
+ Enable debug logging for proxy operations.
547
+ """
548
+ ...
549
+
550
+ @typing.overload
551
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
552
+ """
553
+ Decorator prototype for all step decorators. This function gets specialized
554
+ and imported for all decorators types by _import_plugin_decorators().
555
+ """
556
+ ...
557
+
558
+ @typing.overload
559
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
560
+ ...
561
+
562
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
563
+ """
564
+ Decorator prototype for all step decorators. This function gets specialized
565
+ and imported for all decorators types by _import_plugin_decorators().
566
+ """
567
+ ...
568
+
569
+ @typing.overload
570
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
389
571
  """
390
572
  Enables loading / saving of models within a step.
391
573
 
@@ -445,112 +627,92 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
445
627
  """
446
628
  ...
447
629
 
448
- def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
630
+ @typing.overload
631
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
632
+ ...
633
+
634
+ @typing.overload
635
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
636
+ ...
637
+
638
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
449
639
  """
450
- This decorator is used to run vllm APIs as Metaflow task sidecars.
640
+ Enables loading / saving of models within a step.
451
641
 
452
- User code call
453
- --------------
454
- @vllm(
455
- model="...",
456
- ...
457
- )
642
+ > Examples
643
+ - Saving Models
644
+ ```python
645
+ @model
646
+ @step
647
+ def train(self):
648
+ # current.model.save returns a dictionary reference to the model saved
649
+ self.my_model = current.model.save(
650
+ path_to_my_model,
651
+ label="my_model",
652
+ metadata={
653
+ "epochs": 10,
654
+ "batch-size": 32,
655
+ "learning-rate": 0.001,
656
+ }
657
+ )
658
+ self.next(self.test)
458
659
 
459
- Valid backend options
460
- ---------------------
461
- - 'local': Run as a separate process on the local task machine.
462
-
463
- Valid model options
464
- -------------------
465
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
660
+ @model(load="my_model")
661
+ @step
662
+ def test(self):
663
+ # `current.model.loaded` returns a dictionary of the loaded models
664
+ # where the key is the name of the artifact and the value is the path to the model
665
+ print(os.listdir(current.model.loaded["my_model"]))
666
+ self.next(self.end)
667
+ ```
466
668
 
467
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
468
- If you need multiple models, you must create multiple @vllm decorators.
669
+ - Loading models
670
+ ```python
671
+ @step
672
+ def train(self):
673
+ # current.model.load returns the path to the model loaded
674
+ checkpoint_path = current.model.load(
675
+ self.checkpoint_key,
676
+ )
677
+ model_path = current.model.load(
678
+ self.model,
679
+ )
680
+ self.next(self.test)
681
+ ```
469
682
 
470
683
 
471
684
  Parameters
472
685
  ----------
473
- model: str
474
- HuggingFace model identifier to be served by vLLM.
475
- backend: str
476
- Determines where and how to run the vLLM process.
477
- openai_api_server: bool
478
- Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
479
- Default is False (uses native engine).
480
- Set to True for backward compatibility with existing code.
481
- debug: bool
482
- Whether to turn on verbose debugging logs.
483
- card_refresh_interval: int
484
- Interval in seconds for refreshing the vLLM status card.
485
- Only used when openai_api_server=True.
486
- max_retries: int
487
- Maximum number of retries checking for vLLM server startup.
488
- Only used when openai_api_server=True.
489
- retry_alert_frequency: int
490
- Frequency of alert logs for vLLM server startup retries.
491
- Only used when openai_api_server=True.
492
- engine_args : dict
493
- Additional keyword arguments to pass to the vLLM engine.
494
- For example, `tensor_parallel_size=2`.
686
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
687
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
688
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
689
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
690
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
691
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
692
+
693
+ temp_dir_root : str, default: None
694
+ The root directory under which `current.model.loaded` will store loaded models
495
695
  """
496
696
  ...
497
697
 
498
698
  @typing.overload
499
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
699
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
500
700
  """
501
- Specifies a timeout for your step.
502
-
503
- This decorator is useful if this step may hang indefinitely.
504
-
505
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
506
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
507
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
508
-
509
- Note that all the values specified in parameters are added together so if you specify
510
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
511
-
512
-
513
- Parameters
514
- ----------
515
- seconds : int, default 0
516
- Number of seconds to wait prior to timing out.
517
- minutes : int, default 0
518
- Number of minutes to wait prior to timing out.
519
- hours : int, default 0
520
- Number of hours to wait prior to timing out.
701
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
702
+ It exists to make it easier for users to know that this decorator should only be used with
703
+ a Neo Cloud like Nebius.
521
704
  """
522
705
  ...
523
706
 
524
707
  @typing.overload
525
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
526
- ...
527
-
528
- @typing.overload
529
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
708
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
530
709
  ...
531
710
 
532
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
711
+ def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
533
712
  """
534
- Specifies a timeout for your step.
535
-
536
- This decorator is useful if this step may hang indefinitely.
537
-
538
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
539
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
540
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
541
-
542
- Note that all the values specified in parameters are added together so if you specify
543
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
544
-
545
-
546
- Parameters
547
- ----------
548
- seconds : int, default 0
549
- Number of seconds to wait prior to timing out.
550
- minutes : int, default 0
551
- Number of minutes to wait prior to timing out.
552
- hours : int, default 0
553
- Number of hours to wait prior to timing out.
713
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
714
+ It exists to make it easier for users to know that this decorator should only be used with
715
+ a Neo Cloud like Nebius.
554
716
  """
555
717
  ...
556
718
 
@@ -701,217 +863,63 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
701
863
  """
702
864
  ...
703
865
 
704
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
866
+ @typing.overload
867
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
705
868
  """
706
- Decorator that helps cache, version and store models/datasets from huggingface hub.
707
-
708
- > Examples
709
-
710
- **Usage: creating references of models from huggingface that may be loaded in downstream steps**
711
- ```python
712
- @huggingface_hub
713
- @step
714
- def pull_model_from_huggingface(self):
715
- # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
716
- # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
717
- # with the same `repo_id` in the backend storage, it will not download the model again. The return
718
- # value of the function is a reference to the model in the backend storage.
719
- # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
720
-
721
- self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
722
- self.llama_model = current.huggingface_hub.snapshot_download(
723
- repo_id=self.model_id,
724
- allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
725
- )
726
- self.next(self.train)
727
- ```
728
-
729
- **Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
730
- ```python
731
- @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
732
- @step
733
- def pull_model_from_huggingface(self):
734
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
735
- ```
736
-
737
- ```python
738
- @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
739
- @step
740
- def finetune_model(self):
741
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
742
- # path_to_model will be /my-directory
743
- ```
869
+ A simple decorator that demonstrates using CardDecoratorInjector
870
+ to inject a card and render simple markdown content.
871
+ """
872
+ ...
873
+
874
+ @typing.overload
875
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
876
+ ...
877
+
878
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
879
+ """
880
+ A simple decorator that demonstrates using CardDecoratorInjector
881
+ to inject a card and render simple markdown content.
882
+ """
883
+ ...
884
+
885
+ @typing.overload
886
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
887
+ """
888
+ Specifies the PyPI packages for the step.
744
889
 
745
- ```python
746
- # Takes all the arguments passed to `snapshot_download`
747
- # except for `local_dir`
748
- @huggingface_hub(load=[
749
- {
750
- "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
751
- },
752
- {
753
- "repo_id": "myorg/mistral-lora",
754
- "repo_type": "model",
755
- },
756
- ])
757
- @step
758
- def finetune_model(self):
759
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
760
- # path_to_model will be /my-directory
761
- ```
890
+ Information in this decorator will augment any
891
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
892
+ you can use `@pypi_base` to set packages required by all
893
+ steps and use `@pypi` to specify step-specific overrides.
762
894
 
763
895
 
764
896
  Parameters
765
897
  ----------
766
- temp_dir_root : str, optional
767
- The root directory that will hold the temporary directory where objects will be downloaded.
768
-
769
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
770
- The list of repos (models/datasets) to load.
771
-
772
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
773
-
774
- - If repo (model/dataset) is not found in the datastore:
775
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
776
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
777
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
778
-
779
- - If repo is found in the datastore:
780
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
898
+ packages : Dict[str, str], default: {}
899
+ Packages to use for this step. The key is the name of the package
900
+ and the value is the version to use.
901
+ python : str, optional, default: None
902
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
903
+ that the version used will correspond to the version of the Python interpreter used to start the run.
781
904
  """
782
905
  ...
783
906
 
784
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
907
+ @typing.overload
908
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
909
+ ...
910
+
911
+ @typing.overload
912
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
913
+ ...
914
+
915
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
785
916
  """
786
- Specifies that this step should execute on Kubernetes.
787
-
788
-
789
- Parameters
790
- ----------
791
- cpu : int, default 1
792
- Number of CPUs required for this step. If `@resources` is
793
- also present, the maximum value from all decorators is used.
794
- memory : int, default 4096
795
- Memory size (in MB) required for this step. If
796
- `@resources` is also present, the maximum value from all decorators is
797
- used.
798
- disk : int, default 10240
799
- Disk size (in MB) required for this step. If
800
- `@resources` is also present, the maximum value from all decorators is
801
- used.
802
- image : str, optional, default None
803
- Docker image to use when launching on Kubernetes. If not specified, and
804
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
805
- not, a default Docker image mapping to the current version of Python is used.
806
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
807
- If given, the imagePullPolicy to be applied to the Docker image of the step.
808
- image_pull_secrets: List[str], default []
809
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
810
- Kubernetes image pull secrets to use when pulling container images
811
- in Kubernetes.
812
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
813
- Kubernetes service account to use when launching pod in Kubernetes.
814
- secrets : List[str], optional, default None
815
- Kubernetes secrets to use when launching pod in Kubernetes. These
816
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
817
- in Metaflow configuration.
818
- node_selector: Union[Dict[str,str], str], optional, default None
819
- Kubernetes node selector(s) to apply to the pod running the task.
820
- Can be passed in as a comma separated string of values e.g.
821
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
822
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
823
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
824
- Kubernetes namespace to use when launching pod in Kubernetes.
825
- gpu : int, optional, default None
826
- Number of GPUs required for this step. A value of zero implies that
827
- the scheduled node should not have GPUs.
828
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
829
- The vendor of the GPUs to be used for this step.
830
- tolerations : List[Dict[str,str]], default []
831
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
832
- Kubernetes tolerations to use when launching pod in Kubernetes.
833
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
834
- Kubernetes labels to use when launching pod in Kubernetes.
835
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
836
- Kubernetes annotations to use when launching pod in Kubernetes.
837
- use_tmpfs : bool, default False
838
- This enables an explicit tmpfs mount for this step.
839
- tmpfs_tempdir : bool, default True
840
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
841
- tmpfs_size : int, optional, default: None
842
- The value for the size (in MiB) of the tmpfs mount for this step.
843
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
844
- memory allocated for this step.
845
- tmpfs_path : str, optional, default /metaflow_temp
846
- Path to tmpfs mount for this step.
847
- persistent_volume_claims : Dict[str, str], optional, default None
848
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
849
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
850
- shared_memory: int, optional
851
- Shared memory size (in MiB) required for this step
852
- port: int, optional
853
- Port number to specify in the Kubernetes job object
854
- compute_pool : str, optional, default None
855
- Compute pool to be used for for this step.
856
- If not specified, any accessible compute pool within the perimeter is used.
857
- hostname_resolution_timeout: int, default 10 * 60
858
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
859
- Only applicable when @parallel is used.
860
- qos: str, default: Burstable
861
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
862
-
863
- security_context: Dict[str, Any], optional, default None
864
- Container security context. Applies to the task container. Allows the following keys:
865
- - privileged: bool, optional, default None
866
- - allow_privilege_escalation: bool, optional, default None
867
- - run_as_user: int, optional, default None
868
- - run_as_group: int, optional, default None
869
- - run_as_non_root: bool, optional, default None
870
- """
871
- ...
872
-
873
- @typing.overload
874
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
875
- """
876
- Specifies environment variables to be set prior to the execution of a step.
877
-
878
-
879
- Parameters
880
- ----------
881
- vars : Dict[str, str], default {}
882
- Dictionary of environment variables to set.
883
- """
884
- ...
885
-
886
- @typing.overload
887
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
888
- ...
889
-
890
- @typing.overload
891
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
892
- ...
893
-
894
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
895
- """
896
- Specifies environment variables to be set prior to the execution of a step.
897
-
898
-
899
- Parameters
900
- ----------
901
- vars : Dict[str, str], default {}
902
- Dictionary of environment variables to set.
903
- """
904
- ...
905
-
906
- @typing.overload
907
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
908
- """
909
- Specifies the PyPI packages for the step.
910
-
911
- Information in this decorator will augment any
912
- attributes set in the `@pyi_base` flow-level decorator. Hence,
913
- you can use `@pypi_base` to set packages required by all
914
- steps and use `@pypi` to specify step-specific overrides.
917
+ Specifies the PyPI packages for the step.
918
+
919
+ Information in this decorator will augment any
920
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
921
+ you can use `@pypi_base` to set packages required by all
922
+ steps and use `@pypi` to specify step-specific overrides.
915
923
 
916
924
 
917
925
  Parameters
@@ -926,31 +934,21 @@ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] =
926
934
  ...
927
935
 
928
936
  @typing.overload
929
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
937
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
938
+ """
939
+ Decorator prototype for all step decorators. This function gets specialized
940
+ and imported for all decorators types by _import_plugin_decorators().
941
+ """
930
942
  ...
931
943
 
932
944
  @typing.overload
933
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
945
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
934
946
  ...
935
947
 
936
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
948
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
937
949
  """
938
- Specifies the PyPI packages for the step.
939
-
940
- Information in this decorator will augment any
941
- attributes set in the `@pyi_base` flow-level decorator. Hence,
942
- you can use `@pypi_base` to set packages required by all
943
- steps and use `@pypi` to specify step-specific overrides.
944
-
945
-
946
- Parameters
947
- ----------
948
- packages : Dict[str, str], default: {}
949
- Packages to use for this step. The key is the name of the package
950
- and the value is the version to use.
951
- python : str, optional, default: None
952
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
953
- that the version used will correspond to the version of the Python interpreter used to start the run.
950
+ Decorator prototype for all step decorators. This function gets specialized
951
+ and imported for all decorators types by _import_plugin_decorators().
954
952
  """
955
953
  ...
956
954
 
@@ -970,96 +968,150 @@ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[t
970
968
  """
971
969
  ...
972
970
 
973
- @typing.overload
974
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
971
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
975
972
  """
976
- A simple decorator that demonstrates using CardDecoratorInjector
977
- to inject a card and render simple markdown content.
973
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
974
+
975
+ User code call
976
+ --------------
977
+ @ollama(
978
+ models=[...],
979
+ ...
980
+ )
981
+
982
+ Valid backend options
983
+ ---------------------
984
+ - 'local': Run as a separate process on the local task machine.
985
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
986
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
987
+
988
+ Valid model options
989
+ -------------------
990
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
991
+
992
+
993
+ Parameters
994
+ ----------
995
+ models: list[str]
996
+ List of Ollama containers running models in sidecars.
997
+ backend: str
998
+ Determines where and how to run the Ollama process.
999
+ force_pull: bool
1000
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
1001
+ cache_update_policy: str
1002
+ Cache update policy: "auto", "force", or "never".
1003
+ force_cache_update: bool
1004
+ Simple override for "force" cache update policy.
1005
+ debug: bool
1006
+ Whether to turn on verbose debugging logs.
1007
+ circuit_breaker_config: dict
1008
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
1009
+ timeout_config: dict
1010
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
978
1011
  """
979
1012
  ...
980
1013
 
981
1014
  @typing.overload
982
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
983
- ...
984
-
985
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1015
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
986
1016
  """
987
- A simple decorator that demonstrates using CardDecoratorInjector
988
- to inject a card and render simple markdown content.
1017
+ Specifies environment variables to be set prior to the execution of a step.
1018
+
1019
+
1020
+ Parameters
1021
+ ----------
1022
+ vars : Dict[str, str], default {}
1023
+ Dictionary of environment variables to set.
989
1024
  """
990
1025
  ...
991
1026
 
992
1027
  @typing.overload
993
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
994
- """
995
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
996
- It exists to make it easier for users to know that this decorator should only be used with
997
- a Neo Cloud like Nebius.
998
- """
1028
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
999
1029
  ...
1000
1030
 
1001
1031
  @typing.overload
1002
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1003
- ...
1004
-
1005
- def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1006
- """
1007
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1008
- It exists to make it easier for users to know that this decorator should only be used with
1009
- a Neo Cloud like Nebius.
1010
- """
1032
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1011
1033
  ...
1012
1034
 
1013
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1035
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1014
1036
  """
1015
- Specifies that this step should execute on DGX cloud.
1037
+ Specifies environment variables to be set prior to the execution of a step.
1016
1038
 
1017
1039
 
1018
1040
  Parameters
1019
1041
  ----------
1020
- gpu : int
1021
- Number of GPUs to use.
1022
- gpu_type : str
1023
- Type of Nvidia GPU to use.
1042
+ vars : Dict[str, str], default {}
1043
+ Dictionary of environment variables to set.
1024
1044
  """
1025
1045
  ...
1026
1046
 
1027
1047
  @typing.overload
1028
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1048
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1029
1049
  """
1030
- Specifies secrets to be retrieved and injected as environment variables prior to
1031
- the execution of a step.
1032
-
1033
-
1034
- Parameters
1035
- ----------
1036
- sources : List[Union[str, Dict[str, Any]]], default: []
1037
- List of secret specs, defining how the secrets are to be retrieved
1038
- role : str, optional, default: None
1039
- Role to use for fetching secrets
1050
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1051
+ It exists to make it easier for users to know that this decorator should only be used with
1052
+ a Neo Cloud like CoreWeave.
1040
1053
  """
1041
1054
  ...
1042
1055
 
1043
1056
  @typing.overload
1044
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1057
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1045
1058
  ...
1046
1059
 
1047
- @typing.overload
1048
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1060
+ def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1061
+ """
1062
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1063
+ It exists to make it easier for users to know that this decorator should only be used with
1064
+ a Neo Cloud like CoreWeave.
1065
+ """
1049
1066
  ...
1050
1067
 
1051
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
1068
+ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1052
1069
  """
1053
- Specifies secrets to be retrieved and injected as environment variables prior to
1054
- the execution of a step.
1070
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
1071
+
1072
+ User code call
1073
+ --------------
1074
+ @vllm(
1075
+ model="...",
1076
+ ...
1077
+ )
1078
+
1079
+ Valid backend options
1080
+ ---------------------
1081
+ - 'local': Run as a separate process on the local task machine.
1082
+
1083
+ Valid model options
1084
+ -------------------
1085
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
1086
+
1087
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
1088
+ If you need multiple models, you must create multiple @vllm decorators.
1055
1089
 
1056
1090
 
1057
1091
  Parameters
1058
1092
  ----------
1059
- sources : List[Union[str, Dict[str, Any]]], default: []
1060
- List of secret specs, defining how the secrets are to be retrieved
1061
- role : str, optional, default: None
1062
- Role to use for fetching secrets
1093
+ model: str
1094
+ HuggingFace model identifier to be served by vLLM.
1095
+ backend: str
1096
+ Determines where and how to run the vLLM process.
1097
+ openai_api_server: bool
1098
+ Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
1099
+ Default is False (uses native engine).
1100
+ Set to True for backward compatibility with existing code.
1101
+ debug: bool
1102
+ Whether to turn on verbose debugging logs.
1103
+ card_refresh_interval: int
1104
+ Interval in seconds for refreshing the vLLM status card.
1105
+ Only used when openai_api_server=True.
1106
+ max_retries: int
1107
+ Maximum number of retries checking for vLLM server startup.
1108
+ Only used when openai_api_server=True.
1109
+ retry_alert_frequency: int
1110
+ Frequency of alert logs for vLLM server startup retries.
1111
+ Only used when openai_api_server=True.
1112
+ engine_args : dict
1113
+ Additional keyword arguments to pass to the vLLM engine.
1114
+ For example, `tensor_parallel_size=2`.
1063
1115
  """
1064
1116
  ...
1065
1117
 
@@ -1142,209 +1194,328 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
1142
1194
  """
1143
1195
  ...
1144
1196
 
1145
- @typing.overload
1146
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1147
- """
1148
- Decorator prototype for all step decorators. This function gets specialized
1149
- and imported for all decorators types by _import_plugin_decorators().
1150
- """
1151
- ...
1152
-
1153
- @typing.overload
1154
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1155
- ...
1156
-
1157
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1158
- """
1159
- Decorator prototype for all step decorators. This function gets specialized
1160
- and imported for all decorators types by _import_plugin_decorators().
1161
- """
1162
- ...
1163
-
1164
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1197
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1165
1198
  """
1166
- S3 Proxy decorator for routing S3 requests through a local proxy service.
1199
+ Specifies that this step should execute on Kubernetes.
1167
1200
 
1168
1201
 
1169
1202
  Parameters
1170
1203
  ----------
1171
- integration_name : str, optional
1172
- Name of the S3 proxy integration. If not specified, will use the only
1173
- available S3 proxy integration in the namespace (fails if multiple exist).
1174
- write_mode : str, optional
1175
- The desired behavior during write operations to target (origin) S3 bucket.
1176
- allowed options are:
1177
- "origin-and-cache" -> write to both the target S3 bucket and local object
1178
- storage
1179
- "origin" -> only write to the target S3 bucket
1180
- "cache" -> only write to the object storage service used for caching
1181
- debug : bool, optional
1182
- Enable debug logging for proxy operations.
1183
- """
1184
- ...
1185
-
1186
- @typing.overload
1187
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1188
- """
1189
- Internal decorator to support Fast bakery
1204
+ cpu : int, default 1
1205
+ Number of CPUs required for this step. If `@resources` is
1206
+ also present, the maximum value from all decorators is used.
1207
+ memory : int, default 4096
1208
+ Memory size (in MB) required for this step. If
1209
+ `@resources` is also present, the maximum value from all decorators is
1210
+ used.
1211
+ disk : int, default 10240
1212
+ Disk size (in MB) required for this step. If
1213
+ `@resources` is also present, the maximum value from all decorators is
1214
+ used.
1215
+ image : str, optional, default None
1216
+ Docker image to use when launching on Kubernetes. If not specified, and
1217
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1218
+ not, a default Docker image mapping to the current version of Python is used.
1219
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1220
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
1221
+ image_pull_secrets: List[str], default []
1222
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
1223
+ Kubernetes image pull secrets to use when pulling container images
1224
+ in Kubernetes.
1225
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1226
+ Kubernetes service account to use when launching pod in Kubernetes.
1227
+ secrets : List[str], optional, default None
1228
+ Kubernetes secrets to use when launching pod in Kubernetes. These
1229
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1230
+ in Metaflow configuration.
1231
+ node_selector: Union[Dict[str,str], str], optional, default None
1232
+ Kubernetes node selector(s) to apply to the pod running the task.
1233
+ Can be passed in as a comma separated string of values e.g.
1234
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
1235
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
1236
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1237
+ Kubernetes namespace to use when launching pod in Kubernetes.
1238
+ gpu : int, optional, default None
1239
+ Number of GPUs required for this step. A value of zero implies that
1240
+ the scheduled node should not have GPUs.
1241
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1242
+ The vendor of the GPUs to be used for this step.
1243
+ tolerations : List[Dict[str,str]], default []
1244
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1245
+ Kubernetes tolerations to use when launching pod in Kubernetes.
1246
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
1247
+ Kubernetes labels to use when launching pod in Kubernetes.
1248
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
1249
+ Kubernetes annotations to use when launching pod in Kubernetes.
1250
+ use_tmpfs : bool, default False
1251
+ This enables an explicit tmpfs mount for this step.
1252
+ tmpfs_tempdir : bool, default True
1253
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1254
+ tmpfs_size : int, optional, default: None
1255
+ The value for the size (in MiB) of the tmpfs mount for this step.
1256
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1257
+ memory allocated for this step.
1258
+ tmpfs_path : str, optional, default /metaflow_temp
1259
+ Path to tmpfs mount for this step.
1260
+ persistent_volume_claims : Dict[str, str], optional, default None
1261
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1262
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1263
+ shared_memory: int, optional
1264
+ Shared memory size (in MiB) required for this step
1265
+ port: int, optional
1266
+ Port number to specify in the Kubernetes job object
1267
+ compute_pool : str, optional, default None
1268
+ Compute pool to be used for for this step.
1269
+ If not specified, any accessible compute pool within the perimeter is used.
1270
+ hostname_resolution_timeout: int, default 10 * 60
1271
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
1272
+ Only applicable when @parallel is used.
1273
+ qos: str, default: Burstable
1274
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
1275
+
1276
+ security_context: Dict[str, Any], optional, default None
1277
+ Container security context. Applies to the task container. Allows the following keys:
1278
+ - privileged: bool, optional, default None
1279
+ - allow_privilege_escalation: bool, optional, default None
1280
+ - run_as_user: int, optional, default None
1281
+ - run_as_group: int, optional, default None
1282
+ - run_as_non_root: bool, optional, default None
1190
1283
  """
1191
1284
  ...
1192
1285
 
1193
- @typing.overload
1194
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1195
- ...
1196
-
1197
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1286
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1198
1287
  """
1199
- Internal decorator to support Fast bakery
1288
+ Specifies that this step should execute on DGX cloud.
1289
+
1290
+
1291
+ Parameters
1292
+ ----------
1293
+ gpu : int
1294
+ Number of GPUs to use.
1295
+ gpu_type : str
1296
+ Type of Nvidia GPU to use.
1200
1297
  """
1201
1298
  ...
1202
1299
 
1203
1300
  @typing.overload
1204
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1301
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1205
1302
  """
1206
- Creates a human-readable report, a Metaflow Card, after this step completes.
1303
+ Specifies that the step will success under all circumstances.
1207
1304
 
1208
- Note that you may add multiple `@card` decorators in a step with different parameters.
1305
+ The decorator will create an optional artifact, specified by `var`, which
1306
+ contains the exception raised. You can use it to detect the presence
1307
+ of errors, indicating that all happy-path artifacts produced by the step
1308
+ are missing.
1209
1309
 
1210
1310
 
1211
1311
  Parameters
1212
1312
  ----------
1213
- type : str, default 'default'
1214
- Card type.
1215
- id : str, optional, default None
1216
- If multiple cards are present, use this id to identify this card.
1217
- options : Dict[str, Any], default {}
1218
- Options passed to the card. The contents depend on the card type.
1219
- timeout : int, default 45
1220
- Interrupt reporting if it takes more than this many seconds.
1313
+ var : str, optional, default None
1314
+ Name of the artifact in which to store the caught exception.
1315
+ If not specified, the exception is not stored.
1316
+ print_exception : bool, default True
1317
+ Determines whether or not the exception is printed to
1318
+ stdout when caught.
1221
1319
  """
1222
1320
  ...
1223
1321
 
1224
1322
  @typing.overload
1225
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1323
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1226
1324
  ...
1227
1325
 
1228
1326
  @typing.overload
1229
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1327
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1230
1328
  ...
1231
1329
 
1232
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1330
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1233
1331
  """
1234
- Creates a human-readable report, a Metaflow Card, after this step completes.
1332
+ Specifies that the step will success under all circumstances.
1235
1333
 
1236
- Note that you may add multiple `@card` decorators in a step with different parameters.
1334
+ The decorator will create an optional artifact, specified by `var`, which
1335
+ contains the exception raised. You can use it to detect the presence
1336
+ of errors, indicating that all happy-path artifacts produced by the step
1337
+ are missing.
1237
1338
 
1238
1339
 
1239
1340
  Parameters
1240
1341
  ----------
1241
- type : str, default 'default'
1242
- Card type.
1243
- id : str, optional, default None
1244
- If multiple cards are present, use this id to identify this card.
1245
- options : Dict[str, Any], default {}
1246
- Options passed to the card. The contents depend on the card type.
1247
- timeout : int, default 45
1248
- Interrupt reporting if it takes more than this many seconds.
1249
- """
1250
- ...
1251
-
1252
- @typing.overload
1253
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1254
- """
1255
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1256
- It exists to make it easier for users to know that this decorator should only be used with
1257
- a Neo Cloud like CoreWeave.
1342
+ var : str, optional, default None
1343
+ Name of the artifact in which to store the caught exception.
1344
+ If not specified, the exception is not stored.
1345
+ print_exception : bool, default True
1346
+ Determines whether or not the exception is printed to
1347
+ stdout when caught.
1258
1348
  """
1259
1349
  ...
1260
1350
 
1261
- @typing.overload
1262
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1263
- ...
1264
-
1265
- def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1351
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1266
1352
  """
1267
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1268
- It exists to make it easier for users to know that this decorator should only be used with
1269
- a Neo Cloud like CoreWeave.
1353
+ Specifies what flows belong to the same project.
1354
+
1355
+ A project-specific namespace is created for all flows that
1356
+ use the same `@project(name)`.
1357
+
1358
+
1359
+ Parameters
1360
+ ----------
1361
+ name : str
1362
+ Project name. Make sure that the name is unique amongst all
1363
+ projects that use the same production scheduler. The name may
1364
+ contain only lowercase alphanumeric characters and underscores.
1365
+
1366
+ branch : Optional[str], default None
1367
+ The branch to use. If not specified, the branch is set to
1368
+ `user.<username>` unless `production` is set to `True`. This can
1369
+ also be set on the command line using `--branch` as a top-level option.
1370
+ It is an error to specify `branch` in the decorator and on the command line.
1371
+
1372
+ production : bool, default False
1373
+ Whether or not the branch is the production branch. This can also be set on the
1374
+ command line using `--production` as a top-level option. It is an error to specify
1375
+ `production` in the decorator and on the command line.
1376
+ The project branch name will be:
1377
+ - if `branch` is specified:
1378
+ - if `production` is True: `prod.<branch>`
1379
+ - if `production` is False: `test.<branch>`
1380
+ - if `branch` is not specified:
1381
+ - if `production` is True: `prod`
1382
+ - if `production` is False: `user.<username>`
1270
1383
  """
1271
1384
  ...
1272
1385
 
1273
1386
  @typing.overload
1274
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1387
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1275
1388
  """
1276
- Specifies the Conda environment for the step.
1389
+ Specifies the event(s) that this flow depends on.
1277
1390
 
1278
- Information in this decorator will augment any
1279
- attributes set in the `@conda_base` flow-level decorator. Hence,
1280
- you can use `@conda_base` to set packages required by all
1281
- steps and use `@conda` to specify step-specific overrides.
1391
+ ```
1392
+ @trigger(event='foo')
1393
+ ```
1394
+ or
1395
+ ```
1396
+ @trigger(events=['foo', 'bar'])
1397
+ ```
1398
+
1399
+ Additionally, you can specify the parameter mappings
1400
+ to map event payload to Metaflow parameters for the flow.
1401
+ ```
1402
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1403
+ ```
1404
+ or
1405
+ ```
1406
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1407
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1408
+ ```
1409
+
1410
+ 'parameters' can also be a list of strings and tuples like so:
1411
+ ```
1412
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1413
+ ```
1414
+ This is equivalent to:
1415
+ ```
1416
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1417
+ ```
1282
1418
 
1283
1419
 
1284
1420
  Parameters
1285
1421
  ----------
1286
- packages : Dict[str, str], default {}
1287
- Packages to use for this step. The key is the name of the package
1288
- and the value is the version to use.
1289
- libraries : Dict[str, str], default {}
1290
- Supported for backward compatibility. When used with packages, packages will take precedence.
1291
- python : str, optional, default None
1292
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1293
- that the version used will correspond to the version of the Python interpreter used to start the run.
1294
- disabled : bool, default False
1295
- If set to True, disables @conda.
1422
+ event : Union[str, Dict[str, Any]], optional, default None
1423
+ Event dependency for this flow.
1424
+ events : List[Union[str, Dict[str, Any]]], default []
1425
+ Events dependency for this flow.
1426
+ options : Dict[str, Any], default {}
1427
+ Backend-specific configuration for tuning eventing behavior.
1296
1428
  """
1297
1429
  ...
1298
1430
 
1299
1431
  @typing.overload
1300
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1301
- ...
1302
-
1303
- @typing.overload
1304
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1432
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1305
1433
  ...
1306
1434
 
1307
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1435
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1308
1436
  """
1309
- Specifies the Conda environment for the step.
1437
+ Specifies the event(s) that this flow depends on.
1310
1438
 
1311
- Information in this decorator will augment any
1312
- attributes set in the `@conda_base` flow-level decorator. Hence,
1313
- you can use `@conda_base` to set packages required by all
1314
- steps and use `@conda` to specify step-specific overrides.
1439
+ ```
1440
+ @trigger(event='foo')
1441
+ ```
1442
+ or
1443
+ ```
1444
+ @trigger(events=['foo', 'bar'])
1445
+ ```
1446
+
1447
+ Additionally, you can specify the parameter mappings
1448
+ to map event payload to Metaflow parameters for the flow.
1449
+ ```
1450
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1451
+ ```
1452
+ or
1453
+ ```
1454
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1455
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1456
+ ```
1457
+
1458
+ 'parameters' can also be a list of strings and tuples like so:
1459
+ ```
1460
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1461
+ ```
1462
+ This is equivalent to:
1463
+ ```
1464
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1465
+ ```
1315
1466
 
1316
1467
 
1317
1468
  Parameters
1318
1469
  ----------
1319
- packages : Dict[str, str], default {}
1320
- Packages to use for this step. The key is the name of the package
1321
- and the value is the version to use.
1322
- libraries : Dict[str, str], default {}
1323
- Supported for backward compatibility. When used with packages, packages will take precedence.
1324
- python : str, optional, default None
1325
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1326
- that the version used will correspond to the version of the Python interpreter used to start the run.
1327
- disabled : bool, default False
1328
- If set to True, disables @conda.
1329
- """
1330
- ...
1331
-
1332
- @typing.overload
1333
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1334
- """
1335
- Decorator prototype for all step decorators. This function gets specialized
1336
- and imported for all decorators types by _import_plugin_decorators().
1470
+ event : Union[str, Dict[str, Any]], optional, default None
1471
+ Event dependency for this flow.
1472
+ events : List[Union[str, Dict[str, Any]]], default []
1473
+ Events dependency for this flow.
1474
+ options : Dict[str, Any], default {}
1475
+ Backend-specific configuration for tuning eventing behavior.
1337
1476
  """
1338
1477
  ...
1339
1478
 
1340
- @typing.overload
1341
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1342
- ...
1343
-
1344
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1479
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1345
1480
  """
1346
- Decorator prototype for all step decorators. This function gets specialized
1347
- and imported for all decorators types by _import_plugin_decorators().
1481
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1482
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1483
+
1484
+
1485
+ Parameters
1486
+ ----------
1487
+ timeout : int
1488
+ Time, in seconds before the task times out and fails. (Default: 3600)
1489
+ poke_interval : int
1490
+ Time in seconds that the job should wait in between each try. (Default: 60)
1491
+ mode : str
1492
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1493
+ exponential_backoff : bool
1494
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1495
+ pool : str
1496
+ the slot pool this task should run in,
1497
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1498
+ soft_fail : bool
1499
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1500
+ name : str
1501
+ Name of the sensor on Airflow
1502
+ description : str
1503
+ Description of sensor in the Airflow UI
1504
+ external_dag_id : str
1505
+ The dag_id that contains the task you want to wait for.
1506
+ external_task_ids : List[str]
1507
+ The list of task_ids that you want to wait for.
1508
+ If None (default value) the sensor waits for the DAG. (Default: None)
1509
+ allowed_states : List[str]
1510
+ Iterable of allowed states, (Default: ['success'])
1511
+ failed_states : List[str]
1512
+ Iterable of failed or dis-allowed states. (Default: None)
1513
+ execution_delta : datetime.timedelta
1514
+ time difference with the previous execution to look at,
1515
+ the default is the same logical date as the current task or DAG. (Default: None)
1516
+ check_existence: bool
1517
+ Set to True to check if the external task exists or check if
1518
+ the DAG to wait for exists. (Default: True)
1348
1519
  """
1349
1520
  ...
1350
1521
 
@@ -1462,139 +1633,97 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1462
1633
  """
1463
1634
  ...
1464
1635
 
1465
- @typing.overload
1466
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1636
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1467
1637
  """
1468
- Specifies the flow(s) that this flow depends on.
1469
-
1470
- ```
1471
- @trigger_on_finish(flow='FooFlow')
1472
- ```
1473
- or
1474
- ```
1475
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1476
- ```
1477
- This decorator respects the @project decorator and triggers the flow
1478
- when upstream runs within the same namespace complete successfully
1479
-
1480
- Additionally, you can specify project aware upstream flow dependencies
1481
- by specifying the fully qualified project_flow_name.
1482
- ```
1483
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1484
- ```
1485
- or
1486
- ```
1487
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1488
- ```
1489
-
1490
- You can also specify just the project or project branch (other values will be
1491
- inferred from the current project or project branch):
1492
- ```
1493
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1494
- ```
1495
-
1496
- Note that `branch` is typically one of:
1497
- - `prod`
1498
- - `user.bob`
1499
- - `test.my_experiment`
1500
- - `prod.staging`
1638
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1639
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1640
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1641
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1642
+ starts only after all sensors finish.
1501
1643
 
1502
1644
 
1503
1645
  Parameters
1504
1646
  ----------
1505
- flow : Union[str, Dict[str, str]], optional, default None
1506
- Upstream flow dependency for this flow.
1507
- flows : List[Union[str, Dict[str, str]]], default []
1508
- Upstream flow dependencies for this flow.
1509
- options : Dict[str, Any], default {}
1510
- Backend-specific configuration for tuning eventing behavior.
1647
+ timeout : int
1648
+ Time, in seconds before the task times out and fails. (Default: 3600)
1649
+ poke_interval : int
1650
+ Time in seconds that the job should wait in between each try. (Default: 60)
1651
+ mode : str
1652
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1653
+ exponential_backoff : bool
1654
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1655
+ pool : str
1656
+ the slot pool this task should run in,
1657
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1658
+ soft_fail : bool
1659
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1660
+ name : str
1661
+ Name of the sensor on Airflow
1662
+ description : str
1663
+ Description of sensor in the Airflow UI
1664
+ bucket_key : Union[str, List[str]]
1665
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1666
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1667
+ bucket_name : str
1668
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1669
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1670
+ wildcard_match : bool
1671
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1672
+ aws_conn_id : str
1673
+ a reference to the s3 connection on Airflow. (Default: None)
1674
+ verify : bool
1675
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1511
1676
  """
1512
1677
  ...
1513
1678
 
1514
1679
  @typing.overload
1515
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1516
- ...
1517
-
1518
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1680
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1519
1681
  """
1520
- Specifies the flow(s) that this flow depends on.
1521
-
1522
- ```
1523
- @trigger_on_finish(flow='FooFlow')
1524
- ```
1525
- or
1526
- ```
1527
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1528
- ```
1529
- This decorator respects the @project decorator and triggers the flow
1530
- when upstream runs within the same namespace complete successfully
1531
-
1532
- Additionally, you can specify project aware upstream flow dependencies
1533
- by specifying the fully qualified project_flow_name.
1534
- ```
1535
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1536
- ```
1537
- or
1538
- ```
1539
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1540
- ```
1541
-
1542
- You can also specify just the project or project branch (other values will be
1543
- inferred from the current project or project branch):
1544
- ```
1545
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1546
- ```
1682
+ Specifies the Conda environment for all steps of the flow.
1547
1683
 
1548
- Note that `branch` is typically one of:
1549
- - `prod`
1550
- - `user.bob`
1551
- - `test.my_experiment`
1552
- - `prod.staging`
1684
+ Use `@conda_base` to set common libraries required by all
1685
+ steps and use `@conda` to specify step-specific additions.
1553
1686
 
1554
1687
 
1555
1688
  Parameters
1556
1689
  ----------
1557
- flow : Union[str, Dict[str, str]], optional, default None
1558
- Upstream flow dependency for this flow.
1559
- flows : List[Union[str, Dict[str, str]]], default []
1560
- Upstream flow dependencies for this flow.
1561
- options : Dict[str, Any], default {}
1562
- Backend-specific configuration for tuning eventing behavior.
1690
+ packages : Dict[str, str], default {}
1691
+ Packages to use for this flow. The key is the name of the package
1692
+ and the value is the version to use.
1693
+ libraries : Dict[str, str], default {}
1694
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1695
+ python : str, optional, default None
1696
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1697
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1698
+ disabled : bool, default False
1699
+ If set to True, disables Conda.
1563
1700
  """
1564
1701
  ...
1565
1702
 
1566
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1703
+ @typing.overload
1704
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1705
+ ...
1706
+
1707
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1567
1708
  """
1568
- Specifies what flows belong to the same project.
1709
+ Specifies the Conda environment for all steps of the flow.
1569
1710
 
1570
- A project-specific namespace is created for all flows that
1571
- use the same `@project(name)`.
1711
+ Use `@conda_base` to set common libraries required by all
1712
+ steps and use `@conda` to specify step-specific additions.
1572
1713
 
1573
1714
 
1574
1715
  Parameters
1575
1716
  ----------
1576
- name : str
1577
- Project name. Make sure that the name is unique amongst all
1578
- projects that use the same production scheduler. The name may
1579
- contain only lowercase alphanumeric characters and underscores.
1580
-
1581
- branch : Optional[str], default None
1582
- The branch to use. If not specified, the branch is set to
1583
- `user.<username>` unless `production` is set to `True`. This can
1584
- also be set on the command line using `--branch` as a top-level option.
1585
- It is an error to specify `branch` in the decorator and on the command line.
1586
-
1587
- production : bool, default False
1588
- Whether or not the branch is the production branch. This can also be set on the
1589
- command line using `--production` as a top-level option. It is an error to specify
1590
- `production` in the decorator and on the command line.
1591
- The project branch name will be:
1592
- - if `branch` is specified:
1593
- - if `production` is True: `prod.<branch>`
1594
- - if `production` is False: `test.<branch>`
1595
- - if `branch` is not specified:
1596
- - if `production` is True: `prod`
1597
- - if `production` is False: `user.<username>`
1717
+ packages : Dict[str, str], default {}
1718
+ Packages to use for this flow. The key is the name of the package
1719
+ and the value is the version to use.
1720
+ libraries : Dict[str, str], default {}
1721
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1722
+ python : str, optional, default None
1723
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1724
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1725
+ disabled : bool, default False
1726
+ If set to True, disables Conda.
1598
1727
  """
1599
1728
  ...
1600
1729
 
@@ -1690,231 +1819,102 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1690
1819
  """
1691
1820
  ...
1692
1821
 
1693
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1694
- """
1695
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1696
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1697
-
1698
-
1699
- Parameters
1700
- ----------
1701
- timeout : int
1702
- Time, in seconds before the task times out and fails. (Default: 3600)
1703
- poke_interval : int
1704
- Time in seconds that the job should wait in between each try. (Default: 60)
1705
- mode : str
1706
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1707
- exponential_backoff : bool
1708
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1709
- pool : str
1710
- the slot pool this task should run in,
1711
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1712
- soft_fail : bool
1713
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1714
- name : str
1715
- Name of the sensor on Airflow
1716
- description : str
1717
- Description of sensor in the Airflow UI
1718
- external_dag_id : str
1719
- The dag_id that contains the task you want to wait for.
1720
- external_task_ids : List[str]
1721
- The list of task_ids that you want to wait for.
1722
- If None (default value) the sensor waits for the DAG. (Default: None)
1723
- allowed_states : List[str]
1724
- Iterable of allowed states, (Default: ['success'])
1725
- failed_states : List[str]
1726
- Iterable of failed or dis-allowed states. (Default: None)
1727
- execution_delta : datetime.timedelta
1728
- time difference with the previous execution to look at,
1729
- the default is the same logical date as the current task or DAG. (Default: None)
1730
- check_existence: bool
1731
- Set to True to check if the external task exists or check if
1732
- the DAG to wait for exists. (Default: True)
1733
- """
1734
- ...
1735
-
1736
- @typing.overload
1737
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1738
- """
1739
- Specifies the Conda environment for all steps of the flow.
1740
-
1741
- Use `@conda_base` to set common libraries required by all
1742
- steps and use `@conda` to specify step-specific additions.
1743
-
1744
-
1745
- Parameters
1746
- ----------
1747
- packages : Dict[str, str], default {}
1748
- Packages to use for this flow. The key is the name of the package
1749
- and the value is the version to use.
1750
- libraries : Dict[str, str], default {}
1751
- Supported for backward compatibility. When used with packages, packages will take precedence.
1752
- python : str, optional, default None
1753
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1754
- that the version used will correspond to the version of the Python interpreter used to start the run.
1755
- disabled : bool, default False
1756
- If set to True, disables Conda.
1757
- """
1758
- ...
1759
-
1760
- @typing.overload
1761
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1762
- ...
1763
-
1764
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1765
- """
1766
- Specifies the Conda environment for all steps of the flow.
1767
-
1768
- Use `@conda_base` to set common libraries required by all
1769
- steps and use `@conda` to specify step-specific additions.
1770
-
1771
-
1772
- Parameters
1773
- ----------
1774
- packages : Dict[str, str], default {}
1775
- Packages to use for this flow. The key is the name of the package
1776
- and the value is the version to use.
1777
- libraries : Dict[str, str], default {}
1778
- Supported for backward compatibility. When used with packages, packages will take precedence.
1779
- python : str, optional, default None
1780
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1781
- that the version used will correspond to the version of the Python interpreter used to start the run.
1782
- disabled : bool, default False
1783
- If set to True, disables Conda.
1784
- """
1785
- ...
1786
-
1787
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1788
- """
1789
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1790
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1791
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1792
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1793
- starts only after all sensors finish.
1794
-
1795
-
1796
- Parameters
1797
- ----------
1798
- timeout : int
1799
- Time, in seconds before the task times out and fails. (Default: 3600)
1800
- poke_interval : int
1801
- Time in seconds that the job should wait in between each try. (Default: 60)
1802
- mode : str
1803
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1804
- exponential_backoff : bool
1805
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1806
- pool : str
1807
- the slot pool this task should run in,
1808
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1809
- soft_fail : bool
1810
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1811
- name : str
1812
- Name of the sensor on Airflow
1813
- description : str
1814
- Description of sensor in the Airflow UI
1815
- bucket_key : Union[str, List[str]]
1816
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1817
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1818
- bucket_name : str
1819
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1820
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1821
- wildcard_match : bool
1822
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1823
- aws_conn_id : str
1824
- a reference to the s3 connection on Airflow. (Default: None)
1825
- verify : bool
1826
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1827
- """
1828
- ...
1829
-
1830
1822
  @typing.overload
1831
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1823
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1832
1824
  """
1833
- Specifies the event(s) that this flow depends on.
1825
+ Specifies the flow(s) that this flow depends on.
1834
1826
 
1835
1827
  ```
1836
- @trigger(event='foo')
1828
+ @trigger_on_finish(flow='FooFlow')
1837
1829
  ```
1838
1830
  or
1839
1831
  ```
1840
- @trigger(events=['foo', 'bar'])
1832
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1841
1833
  ```
1834
+ This decorator respects the @project decorator and triggers the flow
1835
+ when upstream runs within the same namespace complete successfully
1842
1836
 
1843
- Additionally, you can specify the parameter mappings
1844
- to map event payload to Metaflow parameters for the flow.
1837
+ Additionally, you can specify project aware upstream flow dependencies
1838
+ by specifying the fully qualified project_flow_name.
1845
1839
  ```
1846
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1840
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1847
1841
  ```
1848
1842
  or
1849
1843
  ```
1850
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1851
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1844
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1852
1845
  ```
1853
1846
 
1854
- 'parameters' can also be a list of strings and tuples like so:
1855
- ```
1856
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1857
- ```
1858
- This is equivalent to:
1847
+ You can also specify just the project or project branch (other values will be
1848
+ inferred from the current project or project branch):
1859
1849
  ```
1860
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1850
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1861
1851
  ```
1862
1852
 
1853
+ Note that `branch` is typically one of:
1854
+ - `prod`
1855
+ - `user.bob`
1856
+ - `test.my_experiment`
1857
+ - `prod.staging`
1858
+
1863
1859
 
1864
1860
  Parameters
1865
1861
  ----------
1866
- event : Union[str, Dict[str, Any]], optional, default None
1867
- Event dependency for this flow.
1868
- events : List[Union[str, Dict[str, Any]]], default []
1869
- Events dependency for this flow.
1862
+ flow : Union[str, Dict[str, str]], optional, default None
1863
+ Upstream flow dependency for this flow.
1864
+ flows : List[Union[str, Dict[str, str]]], default []
1865
+ Upstream flow dependencies for this flow.
1870
1866
  options : Dict[str, Any], default {}
1871
1867
  Backend-specific configuration for tuning eventing behavior.
1872
1868
  """
1873
1869
  ...
1874
1870
 
1875
1871
  @typing.overload
1876
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1872
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1877
1873
  ...
1878
1874
 
1879
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1875
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1880
1876
  """
1881
- Specifies the event(s) that this flow depends on.
1877
+ Specifies the flow(s) that this flow depends on.
1882
1878
 
1883
1879
  ```
1884
- @trigger(event='foo')
1880
+ @trigger_on_finish(flow='FooFlow')
1885
1881
  ```
1886
1882
  or
1887
1883
  ```
1888
- @trigger(events=['foo', 'bar'])
1884
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1889
1885
  ```
1886
+ This decorator respects the @project decorator and triggers the flow
1887
+ when upstream runs within the same namespace complete successfully
1890
1888
 
1891
- Additionally, you can specify the parameter mappings
1892
- to map event payload to Metaflow parameters for the flow.
1889
+ Additionally, you can specify project aware upstream flow dependencies
1890
+ by specifying the fully qualified project_flow_name.
1893
1891
  ```
1894
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1892
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1895
1893
  ```
1896
1894
  or
1897
1895
  ```
1898
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1899
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1896
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1900
1897
  ```
1901
1898
 
1902
- 'parameters' can also be a list of strings and tuples like so:
1903
- ```
1904
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1905
- ```
1906
- This is equivalent to:
1899
+ You can also specify just the project or project branch (other values will be
1900
+ inferred from the current project or project branch):
1907
1901
  ```
1908
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1902
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1909
1903
  ```
1910
1904
 
1905
+ Note that `branch` is typically one of:
1906
+ - `prod`
1907
+ - `user.bob`
1908
+ - `test.my_experiment`
1909
+ - `prod.staging`
1910
+
1911
1911
 
1912
1912
  Parameters
1913
1913
  ----------
1914
- event : Union[str, Dict[str, Any]], optional, default None
1915
- Event dependency for this flow.
1916
- events : List[Union[str, Dict[str, Any]]], default []
1917
- Events dependency for this flow.
1914
+ flow : Union[str, Dict[str, str]], optional, default None
1915
+ Upstream flow dependency for this flow.
1916
+ flows : List[Union[str, Dict[str, str]]], default []
1917
+ Upstream flow dependencies for this flow.
1918
1918
  options : Dict[str, Any], default {}
1919
1919
  Backend-specific configuration for tuning eventing behavior.
1920
1920
  """