ob-metaflow-stubs 6.0.10.7__py2.py3-none-any.whl → 6.0.10.8__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow-stubs might be problematic. Click here for more details.

Files changed (262) hide show
  1. metaflow-stubs/__init__.pyi +1039 -1039
  2. metaflow-stubs/cards.pyi +3 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +5 -5
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +35 -35
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +4 -4
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +3 -3
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +3 -3
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +5 -5
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +2 -2
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +4 -4
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +4 -4
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +4 -4
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +3 -3
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +3 -3
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +3 -3
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +2 -2
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  110. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  111. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  112. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  113. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  114. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
  116. metaflow-stubs/multicore_utils.pyi +2 -2
  117. metaflow-stubs/ob_internal.pyi +2 -2
  118. metaflow-stubs/packaging_sys/__init__.pyi +8 -8
  119. metaflow-stubs/packaging_sys/backend.pyi +4 -4
  120. metaflow-stubs/packaging_sys/distribution_support.pyi +5 -5
  121. metaflow-stubs/packaging_sys/tar_backend.pyi +5 -5
  122. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  123. metaflow-stubs/packaging_sys/v1.pyi +3 -3
  124. metaflow-stubs/parameters.pyi +5 -5
  125. metaflow-stubs/plugins/__init__.pyi +13 -13
  126. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  128. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  129. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  130. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  131. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  132. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  133. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  134. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  135. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  136. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  137. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  138. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  139. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  140. metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
  141. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  142. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  143. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  144. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  146. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  147. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  148. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  150. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  152. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  154. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  155. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  157. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  158. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  159. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  160. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  161. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  162. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  163. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  164. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  165. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  166. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  167. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  168. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  169. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  170. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  171. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  172. metaflow-stubs/plugins/cards/card_modules/components.pyi +83 -3
  173. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  174. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  175. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  176. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  177. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  178. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  179. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  180. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  181. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  182. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  183. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  184. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  185. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  186. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  187. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  188. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  189. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  190. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  191. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  192. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  193. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  194. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  195. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  196. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  197. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  198. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  199. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  200. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  201. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  202. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  203. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  204. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  205. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  206. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  207. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  208. metaflow-stubs/plugins/optuna/__init__.pyi +2 -2
  209. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  210. metaflow-stubs/plugins/perimeters.pyi +2 -2
  211. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  212. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  213. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  214. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  215. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  216. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  217. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  218. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  219. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  220. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  221. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  222. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  223. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  224. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  225. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  226. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  227. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  228. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  229. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  230. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  231. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  232. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  233. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  234. metaflow-stubs/profilers/__init__.pyi +2 -2
  235. metaflow-stubs/pylint_wrapper.pyi +2 -2
  236. metaflow-stubs/runner/__init__.pyi +2 -2
  237. metaflow-stubs/runner/deployer.pyi +34 -34
  238. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  239. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  240. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  241. metaflow-stubs/runner/nbrun.pyi +2 -2
  242. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  243. metaflow-stubs/runner/utils.pyi +2 -2
  244. metaflow-stubs/system/__init__.pyi +2 -2
  245. metaflow-stubs/system/system_logger.pyi +2 -2
  246. metaflow-stubs/system/system_monitor.pyi +2 -2
  247. metaflow-stubs/tagging_util.pyi +2 -2
  248. metaflow-stubs/tuple_util.pyi +2 -2
  249. metaflow-stubs/user_configs/__init__.pyi +2 -2
  250. metaflow-stubs/user_configs/config_options.pyi +3 -3
  251. metaflow-stubs/user_configs/config_parameters.pyi +7 -7
  252. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  253. metaflow-stubs/user_decorators/common.pyi +2 -2
  254. metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
  255. metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
  256. metaflow-stubs/user_decorators/user_flow_decorator.pyi +4 -4
  257. metaflow-stubs/user_decorators/user_step_decorator.pyi +6 -6
  258. {ob_metaflow_stubs-6.0.10.7.dist-info → ob_metaflow_stubs-6.0.10.8.dist-info}/METADATA +1 -1
  259. ob_metaflow_stubs-6.0.10.8.dist-info/RECORD +262 -0
  260. ob_metaflow_stubs-6.0.10.7.dist-info/RECORD +0 -262
  261. {ob_metaflow_stubs-6.0.10.7.dist-info → ob_metaflow_stubs-6.0.10.8.dist-info}/WHEEL +0 -0
  262. {ob_metaflow_stubs-6.0.10.7.dist-info → ob_metaflow_stubs-6.0.10.8.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.18.7.2+obcheckpoint(0.2.6);ob(v1) #
4
- # Generated on 2025-09-19T18:41:10.791351 #
3
+ # MF version: 2.18.7.5+obcheckpoint(0.2.6);ob(v1) #
4
+ # Generated on 2025-09-19T21:56:58.875223 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -39,10 +39,10 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
39
39
  from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
- from . import cards as cards
43
42
  from . import metaflow_git as metaflow_git
44
- from . import events as events
45
43
  from . import tuple_util as tuple_util
44
+ from . import cards as cards
45
+ from . import events as events
46
46
  from . import runner as runner
47
47
  from . import plugins as plugins
48
48
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
@@ -168,494 +168,158 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
168
168
  ...
169
169
 
170
170
  @typing.overload
171
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
171
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
172
172
  """
173
- Specifies the number of times the task corresponding
174
- to a step needs to be retried.
175
-
176
- This decorator is useful for handling transient errors, such as networking issues.
177
- If your task contains operations that can't be retried safely, e.g. database updates,
178
- it is advisable to annotate it with `@retry(times=0)`.
179
-
180
- This can be used in conjunction with the `@catch` decorator. The `@catch`
181
- decorator will execute a no-op task after all retries have been exhausted,
182
- ensuring that the flow execution can continue.
183
-
184
-
185
- Parameters
186
- ----------
187
- times : int, default 3
188
- Number of times to retry this task.
189
- minutes_between_retries : int, default 2
190
- Number of minutes between retries.
173
+ Decorator prototype for all step decorators. This function gets specialized
174
+ and imported for all decorators types by _import_plugin_decorators().
191
175
  """
192
176
  ...
193
177
 
194
178
  @typing.overload
195
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
196
- ...
197
-
198
- @typing.overload
199
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
179
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
200
180
  ...
201
181
 
202
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
182
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
203
183
  """
204
- Specifies the number of times the task corresponding
205
- to a step needs to be retried.
206
-
207
- This decorator is useful for handling transient errors, such as networking issues.
208
- If your task contains operations that can't be retried safely, e.g. database updates,
209
- it is advisable to annotate it with `@retry(times=0)`.
210
-
211
- This can be used in conjunction with the `@catch` decorator. The `@catch`
212
- decorator will execute a no-op task after all retries have been exhausted,
213
- ensuring that the flow execution can continue.
214
-
215
-
216
- Parameters
217
- ----------
218
- times : int, default 3
219
- Number of times to retry this task.
220
- minutes_between_retries : int, default 2
221
- Number of minutes between retries.
184
+ Decorator prototype for all step decorators. This function gets specialized
185
+ and imported for all decorators types by _import_plugin_decorators().
222
186
  """
223
187
  ...
224
188
 
225
- def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
189
+ @typing.overload
190
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
226
191
  """
227
- This decorator is used to run vllm APIs as Metaflow task sidecars.
228
-
229
- User code call
230
- --------------
231
- @vllm(
232
- model="...",
233
- ...
234
- )
192
+ Specifies a timeout for your step.
235
193
 
236
- Valid backend options
237
- ---------------------
238
- - 'local': Run as a separate process on the local task machine.
194
+ This decorator is useful if this step may hang indefinitely.
239
195
 
240
- Valid model options
241
- -------------------
242
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
196
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
197
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
198
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
243
199
 
244
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
245
- If you need multiple models, you must create multiple @vllm decorators.
200
+ Note that all the values specified in parameters are added together so if you specify
201
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
246
202
 
247
203
 
248
204
  Parameters
249
205
  ----------
250
- model: str
251
- HuggingFace model identifier to be served by vLLM.
252
- backend: str
253
- Determines where and how to run the vLLM process.
254
- openai_api_server: bool
255
- Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
256
- Default is False (uses native engine).
257
- Set to True for backward compatibility with existing code.
258
- debug: bool
259
- Whether to turn on verbose debugging logs.
260
- card_refresh_interval: int
261
- Interval in seconds for refreshing the vLLM status card.
262
- Only used when openai_api_server=True.
263
- max_retries: int
264
- Maximum number of retries checking for vLLM server startup.
265
- Only used when openai_api_server=True.
266
- retry_alert_frequency: int
267
- Frequency of alert logs for vLLM server startup retries.
268
- Only used when openai_api_server=True.
269
- engine_args : dict
270
- Additional keyword arguments to pass to the vLLM engine.
271
- For example, `tensor_parallel_size=2`.
206
+ seconds : int, default 0
207
+ Number of seconds to wait prior to timing out.
208
+ minutes : int, default 0
209
+ Number of minutes to wait prior to timing out.
210
+ hours : int, default 0
211
+ Number of hours to wait prior to timing out.
272
212
  """
273
213
  ...
274
214
 
275
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
276
- """
277
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
278
-
279
- User code call
280
- --------------
281
- @ollama(
282
- models=[...],
283
- ...
284
- )
285
-
286
- Valid backend options
287
- ---------------------
288
- - 'local': Run as a separate process on the local task machine.
289
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
290
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
291
-
292
- Valid model options
293
- -------------------
294
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
295
-
296
-
297
- Parameters
298
- ----------
299
- models: list[str]
300
- List of Ollama containers running models in sidecars.
301
- backend: str
302
- Determines where and how to run the Ollama process.
303
- force_pull: bool
304
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
305
- cache_update_policy: str
306
- Cache update policy: "auto", "force", or "never".
307
- force_cache_update: bool
308
- Simple override for "force" cache update policy.
309
- debug: bool
310
- Whether to turn on verbose debugging logs.
311
- circuit_breaker_config: dict
312
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
313
- timeout_config: dict
314
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
315
- """
215
+ @typing.overload
216
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
316
217
  ...
317
218
 
318
219
  @typing.overload
319
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
220
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
221
+ ...
222
+
223
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
320
224
  """
321
- Specifies secrets to be retrieved and injected as environment variables prior to
322
- the execution of a step.
225
+ Specifies a timeout for your step.
226
+
227
+ This decorator is useful if this step may hang indefinitely.
228
+
229
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
230
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
231
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
232
+
233
+ Note that all the values specified in parameters are added together so if you specify
234
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
323
235
 
324
236
 
325
237
  Parameters
326
238
  ----------
327
- sources : List[Union[str, Dict[str, Any]]], default: []
328
- List of secret specs, defining how the secrets are to be retrieved
329
- role : str, optional, default: None
330
- Role to use for fetching secrets
239
+ seconds : int, default 0
240
+ Number of seconds to wait prior to timing out.
241
+ minutes : int, default 0
242
+ Number of minutes to wait prior to timing out.
243
+ hours : int, default 0
244
+ Number of hours to wait prior to timing out.
331
245
  """
332
246
  ...
333
247
 
334
- @typing.overload
335
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
336
- ...
337
-
338
- @typing.overload
339
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
340
- ...
341
-
342
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
248
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
343
249
  """
344
- Specifies secrets to be retrieved and injected as environment variables prior to
345
- the execution of a step.
250
+ Specifies that this step should execute on DGX cloud.
346
251
 
347
252
 
348
253
  Parameters
349
254
  ----------
350
- sources : List[Union[str, Dict[str, Any]]], default: []
351
- List of secret specs, defining how the secrets are to be retrieved
352
- role : str, optional, default: None
353
- Role to use for fetching secrets
255
+ gpu : int
256
+ Number of GPUs to use.
257
+ gpu_type : str
258
+ Type of Nvidia GPU to use.
354
259
  """
355
260
  ...
356
261
 
357
262
  @typing.overload
358
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
263
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
359
264
  """
360
- Specifies the PyPI packages for the step.
265
+ Specifies the Conda environment for the step.
361
266
 
362
267
  Information in this decorator will augment any
363
- attributes set in the `@pyi_base` flow-level decorator. Hence,
364
- you can use `@pypi_base` to set packages required by all
365
- steps and use `@pypi` to specify step-specific overrides.
268
+ attributes set in the `@conda_base` flow-level decorator. Hence,
269
+ you can use `@conda_base` to set packages required by all
270
+ steps and use `@conda` to specify step-specific overrides.
366
271
 
367
272
 
368
273
  Parameters
369
274
  ----------
370
- packages : Dict[str, str], default: {}
275
+ packages : Dict[str, str], default {}
371
276
  Packages to use for this step. The key is the name of the package
372
277
  and the value is the version to use.
373
- python : str, optional, default: None
278
+ libraries : Dict[str, str], default {}
279
+ Supported for backward compatibility. When used with packages, packages will take precedence.
280
+ python : str, optional, default None
374
281
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
375
282
  that the version used will correspond to the version of the Python interpreter used to start the run.
283
+ disabled : bool, default False
284
+ If set to True, disables @conda.
376
285
  """
377
286
  ...
378
287
 
379
288
  @typing.overload
380
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
289
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
381
290
  ...
382
291
 
383
292
  @typing.overload
384
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
293
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
385
294
  ...
386
295
 
387
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
296
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
388
297
  """
389
- Specifies the PyPI packages for the step.
298
+ Specifies the Conda environment for the step.
390
299
 
391
300
  Information in this decorator will augment any
392
- attributes set in the `@pyi_base` flow-level decorator. Hence,
393
- you can use `@pypi_base` to set packages required by all
394
- steps and use `@pypi` to specify step-specific overrides.
301
+ attributes set in the `@conda_base` flow-level decorator. Hence,
302
+ you can use `@conda_base` to set packages required by all
303
+ steps and use `@conda` to specify step-specific overrides.
395
304
 
396
305
 
397
306
  Parameters
398
307
  ----------
399
- packages : Dict[str, str], default: {}
308
+ packages : Dict[str, str], default {}
400
309
  Packages to use for this step. The key is the name of the package
401
310
  and the value is the version to use.
402
- python : str, optional, default: None
311
+ libraries : Dict[str, str], default {}
312
+ Supported for backward compatibility. When used with packages, packages will take precedence.
313
+ python : str, optional, default None
403
314
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
404
315
  that the version used will correspond to the version of the Python interpreter used to start the run.
316
+ disabled : bool, default False
317
+ If set to True, disables @conda.
405
318
  """
406
319
  ...
407
320
 
408
321
  @typing.overload
409
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
410
- """
411
- Internal decorator to support Fast bakery
412
- """
413
- ...
414
-
415
- @typing.overload
416
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
417
- ...
418
-
419
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
420
- """
421
- Internal decorator to support Fast bakery
422
- """
423
- ...
424
-
425
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
426
- """
427
- Specifies that this step should execute on DGX cloud.
428
-
429
-
430
- Parameters
431
- ----------
432
- gpu : int
433
- Number of GPUs to use.
434
- gpu_type : str
435
- Type of Nvidia GPU to use.
436
- """
437
- ...
438
-
439
- @typing.overload
440
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
441
- """
442
- Decorator prototype for all step decorators. This function gets specialized
443
- and imported for all decorators types by _import_plugin_decorators().
444
- """
445
- ...
446
-
447
- @typing.overload
448
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
449
- ...
450
-
451
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
452
- """
453
- Decorator prototype for all step decorators. This function gets specialized
454
- and imported for all decorators types by _import_plugin_decorators().
455
- """
456
- ...
457
-
458
- @typing.overload
459
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
460
- """
461
- A simple decorator that demonstrates using CardDecoratorInjector
462
- to inject a card and render simple markdown content.
463
- """
464
- ...
465
-
466
- @typing.overload
467
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
468
- ...
469
-
470
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
471
- """
472
- A simple decorator that demonstrates using CardDecoratorInjector
473
- to inject a card and render simple markdown content.
474
- """
475
- ...
476
-
477
- @typing.overload
478
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
479
- """
480
- Enables checkpointing for a step.
481
-
482
- > Examples
483
-
484
- - Saving Checkpoints
485
-
486
- ```python
487
- @checkpoint
488
- @step
489
- def train(self):
490
- model = create_model(self.parameters, checkpoint_path = None)
491
- for i in range(self.epochs):
492
- # some training logic
493
- loss = model.train(self.dataset)
494
- if i % 10 == 0:
495
- model.save(
496
- current.checkpoint.directory,
497
- )
498
- # saves the contents of the `current.checkpoint.directory` as a checkpoint
499
- # and returns a reference dictionary to the checkpoint saved in the datastore
500
- self.latest_checkpoint = current.checkpoint.save(
501
- name="epoch_checkpoint",
502
- metadata={
503
- "epoch": i,
504
- "loss": loss,
505
- }
506
- )
507
- ```
508
-
509
- - Using Loaded Checkpoints
510
-
511
- ```python
512
- @retry(times=3)
513
- @checkpoint
514
- @step
515
- def train(self):
516
- # Assume that the task has restarted and the previous attempt of the task
517
- # saved a checkpoint
518
- checkpoint_path = None
519
- if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
520
- print("Loaded checkpoint from the previous attempt")
521
- checkpoint_path = current.checkpoint.directory
522
-
523
- model = create_model(self.parameters, checkpoint_path = checkpoint_path)
524
- for i in range(self.epochs):
525
- ...
526
- ```
527
-
528
-
529
- Parameters
530
- ----------
531
- load_policy : str, default: "fresh"
532
- The policy for loading the checkpoint. The following policies are supported:
533
- - "eager": Loads the the latest available checkpoint within the namespace.
534
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
535
- will be loaded at the start of the task.
536
- - "none": Do not load any checkpoint
537
- - "fresh": Loads the lastest checkpoint created within the running Task.
538
- This mode helps loading checkpoints across various retry attempts of the same task.
539
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
540
- created within the task will be loaded when the task is retries execution on failure.
541
-
542
- temp_dir_root : str, default: None
543
- The root directory under which `current.checkpoint.directory` will be created.
544
- """
545
- ...
546
-
547
- @typing.overload
548
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
549
- ...
550
-
551
- @typing.overload
552
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
553
- ...
554
-
555
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
556
- """
557
- Enables checkpointing for a step.
558
-
559
- > Examples
560
-
561
- - Saving Checkpoints
562
-
563
- ```python
564
- @checkpoint
565
- @step
566
- def train(self):
567
- model = create_model(self.parameters, checkpoint_path = None)
568
- for i in range(self.epochs):
569
- # some training logic
570
- loss = model.train(self.dataset)
571
- if i % 10 == 0:
572
- model.save(
573
- current.checkpoint.directory,
574
- )
575
- # saves the contents of the `current.checkpoint.directory` as a checkpoint
576
- # and returns a reference dictionary to the checkpoint saved in the datastore
577
- self.latest_checkpoint = current.checkpoint.save(
578
- name="epoch_checkpoint",
579
- metadata={
580
- "epoch": i,
581
- "loss": loss,
582
- }
583
- )
584
- ```
585
-
586
- - Using Loaded Checkpoints
587
-
588
- ```python
589
- @retry(times=3)
590
- @checkpoint
591
- @step
592
- def train(self):
593
- # Assume that the task has restarted and the previous attempt of the task
594
- # saved a checkpoint
595
- checkpoint_path = None
596
- if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
597
- print("Loaded checkpoint from the previous attempt")
598
- checkpoint_path = current.checkpoint.directory
599
-
600
- model = create_model(self.parameters, checkpoint_path = checkpoint_path)
601
- for i in range(self.epochs):
602
- ...
603
- ```
604
-
605
-
606
- Parameters
607
- ----------
608
- load_policy : str, default: "fresh"
609
- The policy for loading the checkpoint. The following policies are supported:
610
- - "eager": Loads the the latest available checkpoint within the namespace.
611
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
612
- will be loaded at the start of the task.
613
- - "none": Do not load any checkpoint
614
- - "fresh": Loads the lastest checkpoint created within the running Task.
615
- This mode helps loading checkpoints across various retry attempts of the same task.
616
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
617
- created within the task will be loaded when the task is retries execution on failure.
618
-
619
- temp_dir_root : str, default: None
620
- The root directory under which `current.checkpoint.directory` will be created.
621
- """
622
- ...
623
-
624
- @typing.overload
625
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
626
- """
627
- Specifies environment variables to be set prior to the execution of a step.
628
-
629
-
630
- Parameters
631
- ----------
632
- vars : Dict[str, str], default {}
633
- Dictionary of environment variables to set.
634
- """
635
- ...
636
-
637
- @typing.overload
638
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
639
- ...
640
-
641
- @typing.overload
642
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
643
- ...
644
-
645
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
646
- """
647
- Specifies environment variables to be set prior to the execution of a step.
648
-
649
-
650
- Parameters
651
- ----------
652
- vars : Dict[str, str], default {}
653
- Dictionary of environment variables to set.
654
- """
655
- ...
656
-
657
- @typing.overload
658
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
322
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
659
323
  """
660
324
  Specifies the resources needed when executing this step.
661
325
 
@@ -734,61 +398,518 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
734
398
  ...
735
399
 
736
400
  @typing.overload
737
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
401
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
738
402
  """
739
- Specifies the Conda environment for the step.
740
-
741
- Information in this decorator will augment any
742
- attributes set in the `@conda_base` flow-level decorator. Hence,
743
- you can use `@conda_base` to set packages required by all
744
- steps and use `@conda` to specify step-specific overrides.
745
-
403
+ Enables loading / saving of models within a step.
746
404
 
747
- Parameters
748
- ----------
749
- packages : Dict[str, str], default {}
750
- Packages to use for this step. The key is the name of the package
751
- and the value is the version to use.
752
- libraries : Dict[str, str], default {}
753
- Supported for backward compatibility. When used with packages, packages will take precedence.
754
- python : str, optional, default None
755
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
756
- that the version used will correspond to the version of the Python interpreter used to start the run.
757
- disabled : bool, default False
758
- If set to True, disables @conda.
405
+ > Examples
406
+ - Saving Models
407
+ ```python
408
+ @model
409
+ @step
410
+ def train(self):
411
+ # current.model.save returns a dictionary reference to the model saved
412
+ self.my_model = current.model.save(
413
+ path_to_my_model,
414
+ label="my_model",
415
+ metadata={
416
+ "epochs": 10,
417
+ "batch-size": 32,
418
+ "learning-rate": 0.001,
419
+ }
420
+ )
421
+ self.next(self.test)
422
+
423
+ @model(load="my_model")
424
+ @step
425
+ def test(self):
426
+ # `current.model.loaded` returns a dictionary of the loaded models
427
+ # where the key is the name of the artifact and the value is the path to the model
428
+ print(os.listdir(current.model.loaded["my_model"]))
429
+ self.next(self.end)
430
+ ```
431
+
432
+ - Loading models
433
+ ```python
434
+ @step
435
+ def train(self):
436
+ # current.model.load returns the path to the model loaded
437
+ checkpoint_path = current.model.load(
438
+ self.checkpoint_key,
439
+ )
440
+ model_path = current.model.load(
441
+ self.model,
442
+ )
443
+ self.next(self.test)
444
+ ```
445
+
446
+
447
+ Parameters
448
+ ----------
449
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
450
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
451
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
452
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
453
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
454
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
455
+
456
+ temp_dir_root : str, default: None
457
+ The root directory under which `current.model.loaded` will store loaded models
458
+ """
459
+ ...
460
+
461
+ @typing.overload
462
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
463
+ ...
464
+
465
+ @typing.overload
466
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
467
+ ...
468
+
469
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
470
+ """
471
+ Enables loading / saving of models within a step.
472
+
473
+ > Examples
474
+ - Saving Models
475
+ ```python
476
+ @model
477
+ @step
478
+ def train(self):
479
+ # current.model.save returns a dictionary reference to the model saved
480
+ self.my_model = current.model.save(
481
+ path_to_my_model,
482
+ label="my_model",
483
+ metadata={
484
+ "epochs": 10,
485
+ "batch-size": 32,
486
+ "learning-rate": 0.001,
487
+ }
488
+ )
489
+ self.next(self.test)
490
+
491
+ @model(load="my_model")
492
+ @step
493
+ def test(self):
494
+ # `current.model.loaded` returns a dictionary of the loaded models
495
+ # where the key is the name of the artifact and the value is the path to the model
496
+ print(os.listdir(current.model.loaded["my_model"]))
497
+ self.next(self.end)
498
+ ```
499
+
500
+ - Loading models
501
+ ```python
502
+ @step
503
+ def train(self):
504
+ # current.model.load returns the path to the model loaded
505
+ checkpoint_path = current.model.load(
506
+ self.checkpoint_key,
507
+ )
508
+ model_path = current.model.load(
509
+ self.model,
510
+ )
511
+ self.next(self.test)
512
+ ```
513
+
514
+
515
+ Parameters
516
+ ----------
517
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
518
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
519
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
520
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
521
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
522
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
523
+
524
+ temp_dir_root : str, default: None
525
+ The root directory under which `current.model.loaded` will store loaded models
526
+ """
527
+ ...
528
+
529
+ @typing.overload
530
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
531
+ """
532
+ Specifies that the step will success under all circumstances.
533
+
534
+ The decorator will create an optional artifact, specified by `var`, which
535
+ contains the exception raised. You can use it to detect the presence
536
+ of errors, indicating that all happy-path artifacts produced by the step
537
+ are missing.
538
+
539
+
540
+ Parameters
541
+ ----------
542
+ var : str, optional, default None
543
+ Name of the artifact in which to store the caught exception.
544
+ If not specified, the exception is not stored.
545
+ print_exception : bool, default True
546
+ Determines whether or not the exception is printed to
547
+ stdout when caught.
548
+ """
549
+ ...
550
+
551
+ @typing.overload
552
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
553
+ ...
554
+
555
+ @typing.overload
556
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
557
+ ...
558
+
559
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
560
+ """
561
+ Specifies that the step will success under all circumstances.
562
+
563
+ The decorator will create an optional artifact, specified by `var`, which
564
+ contains the exception raised. You can use it to detect the presence
565
+ of errors, indicating that all happy-path artifacts produced by the step
566
+ are missing.
567
+
568
+
569
+ Parameters
570
+ ----------
571
+ var : str, optional, default None
572
+ Name of the artifact in which to store the caught exception.
573
+ If not specified, the exception is not stored.
574
+ print_exception : bool, default True
575
+ Determines whether or not the exception is printed to
576
+ stdout when caught.
577
+ """
578
+ ...
579
+
580
+ @typing.overload
581
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
582
+ """
583
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
584
+ It exists to make it easier for users to know that this decorator should only be used with
585
+ a Neo Cloud like CoreWeave.
586
+ """
587
+ ...
588
+
589
+ @typing.overload
590
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
591
+ ...
592
+
593
+ def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
594
+ """
595
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
596
+ It exists to make it easier for users to know that this decorator should only be used with
597
+ a Neo Cloud like CoreWeave.
598
+ """
599
+ ...
600
+
601
+ @typing.overload
602
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
603
+ """
604
+ Decorator prototype for all step decorators. This function gets specialized
605
+ and imported for all decorators types by _import_plugin_decorators().
606
+ """
607
+ ...
608
+
609
+ @typing.overload
610
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
611
+ ...
612
+
613
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
614
+ """
615
+ Decorator prototype for all step decorators. This function gets specialized
616
+ and imported for all decorators types by _import_plugin_decorators().
617
+ """
618
+ ...
619
+
620
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
621
+ """
622
+ S3 Proxy decorator for routing S3 requests through a local proxy service.
623
+
624
+
625
+ Parameters
626
+ ----------
627
+ integration_name : str, optional
628
+ Name of the S3 proxy integration. If not specified, will use the only
629
+ available S3 proxy integration in the namespace (fails if multiple exist).
630
+ write_mode : str, optional
631
+ The desired behavior during write operations to target (origin) S3 bucket.
632
+ allowed options are:
633
+ "origin-and-cache" -> write to both the target S3 bucket and local object
634
+ storage
635
+ "origin" -> only write to the target S3 bucket
636
+ "cache" -> only write to the object storage service used for caching
637
+ debug : bool, optional
638
+ Enable debug logging for proxy operations.
639
+ """
640
+ ...
641
+
642
+ @typing.overload
643
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
644
+ """
645
+ Specifies environment variables to be set prior to the execution of a step.
646
+
647
+
648
+ Parameters
649
+ ----------
650
+ vars : Dict[str, str], default {}
651
+ Dictionary of environment variables to set.
652
+ """
653
+ ...
654
+
655
+ @typing.overload
656
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
657
+ ...
658
+
659
+ @typing.overload
660
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
661
+ ...
662
+
663
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
664
+ """
665
+ Specifies environment variables to be set prior to the execution of a step.
666
+
667
+
668
+ Parameters
669
+ ----------
670
+ vars : Dict[str, str], default {}
671
+ Dictionary of environment variables to set.
672
+ """
673
+ ...
674
+
675
+ @typing.overload
676
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
677
+ """
678
+ Enables checkpointing for a step.
679
+
680
+ > Examples
681
+
682
+ - Saving Checkpoints
683
+
684
+ ```python
685
+ @checkpoint
686
+ @step
687
+ def train(self):
688
+ model = create_model(self.parameters, checkpoint_path = None)
689
+ for i in range(self.epochs):
690
+ # some training logic
691
+ loss = model.train(self.dataset)
692
+ if i % 10 == 0:
693
+ model.save(
694
+ current.checkpoint.directory,
695
+ )
696
+ # saves the contents of the `current.checkpoint.directory` as a checkpoint
697
+ # and returns a reference dictionary to the checkpoint saved in the datastore
698
+ self.latest_checkpoint = current.checkpoint.save(
699
+ name="epoch_checkpoint",
700
+ metadata={
701
+ "epoch": i,
702
+ "loss": loss,
703
+ }
704
+ )
705
+ ```
706
+
707
+ - Using Loaded Checkpoints
708
+
709
+ ```python
710
+ @retry(times=3)
711
+ @checkpoint
712
+ @step
713
+ def train(self):
714
+ # Assume that the task has restarted and the previous attempt of the task
715
+ # saved a checkpoint
716
+ checkpoint_path = None
717
+ if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
718
+ print("Loaded checkpoint from the previous attempt")
719
+ checkpoint_path = current.checkpoint.directory
720
+
721
+ model = create_model(self.parameters, checkpoint_path = checkpoint_path)
722
+ for i in range(self.epochs):
723
+ ...
724
+ ```
725
+
726
+
727
+ Parameters
728
+ ----------
729
+ load_policy : str, default: "fresh"
730
+ The policy for loading the checkpoint. The following policies are supported:
731
+ - "eager": Loads the the latest available checkpoint within the namespace.
732
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
733
+ will be loaded at the start of the task.
734
+ - "none": Do not load any checkpoint
735
+ - "fresh": Loads the lastest checkpoint created within the running Task.
736
+ This mode helps loading checkpoints across various retry attempts of the same task.
737
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
738
+ created within the task will be loaded when the task is retries execution on failure.
739
+
740
+ temp_dir_root : str, default: None
741
+ The root directory under which `current.checkpoint.directory` will be created.
759
742
  """
760
743
  ...
761
744
 
762
745
  @typing.overload
763
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
746
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
764
747
  ...
765
748
 
766
749
  @typing.overload
767
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
750
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
768
751
  ...
769
752
 
770
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
753
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
771
754
  """
772
- Specifies the Conda environment for the step.
755
+ Enables checkpointing for a step.
773
756
 
774
- Information in this decorator will augment any
775
- attributes set in the `@conda_base` flow-level decorator. Hence,
776
- you can use `@conda_base` to set packages required by all
777
- steps and use `@conda` to specify step-specific overrides.
757
+ > Examples
758
+
759
+ - Saving Checkpoints
760
+
761
+ ```python
762
+ @checkpoint
763
+ @step
764
+ def train(self):
765
+ model = create_model(self.parameters, checkpoint_path = None)
766
+ for i in range(self.epochs):
767
+ # some training logic
768
+ loss = model.train(self.dataset)
769
+ if i % 10 == 0:
770
+ model.save(
771
+ current.checkpoint.directory,
772
+ )
773
+ # saves the contents of the `current.checkpoint.directory` as a checkpoint
774
+ # and returns a reference dictionary to the checkpoint saved in the datastore
775
+ self.latest_checkpoint = current.checkpoint.save(
776
+ name="epoch_checkpoint",
777
+ metadata={
778
+ "epoch": i,
779
+ "loss": loss,
780
+ }
781
+ )
782
+ ```
783
+
784
+ - Using Loaded Checkpoints
785
+
786
+ ```python
787
+ @retry(times=3)
788
+ @checkpoint
789
+ @step
790
+ def train(self):
791
+ # Assume that the task has restarted and the previous attempt of the task
792
+ # saved a checkpoint
793
+ checkpoint_path = None
794
+ if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
795
+ print("Loaded checkpoint from the previous attempt")
796
+ checkpoint_path = current.checkpoint.directory
797
+
798
+ model = create_model(self.parameters, checkpoint_path = checkpoint_path)
799
+ for i in range(self.epochs):
800
+ ...
801
+ ```
778
802
 
779
803
 
780
804
  Parameters
781
805
  ----------
782
- packages : Dict[str, str], default {}
783
- Packages to use for this step. The key is the name of the package
784
- and the value is the version to use.
785
- libraries : Dict[str, str], default {}
786
- Supported for backward compatibility. When used with packages, packages will take precedence.
787
- python : str, optional, default None
788
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
789
- that the version used will correspond to the version of the Python interpreter used to start the run.
790
- disabled : bool, default False
791
- If set to True, disables @conda.
806
+ load_policy : str, default: "fresh"
807
+ The policy for loading the checkpoint. The following policies are supported:
808
+ - "eager": Loads the the latest available checkpoint within the namespace.
809
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
810
+ will be loaded at the start of the task.
811
+ - "none": Do not load any checkpoint
812
+ - "fresh": Loads the lastest checkpoint created within the running Task.
813
+ This mode helps loading checkpoints across various retry attempts of the same task.
814
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
815
+ created within the task will be loaded when the task is retries execution on failure.
816
+
817
+ temp_dir_root : str, default: None
818
+ The root directory under which `current.checkpoint.directory` will be created.
819
+ """
820
+ ...
821
+
822
+ @typing.overload
823
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
824
+ """
825
+ Specifies secrets to be retrieved and injected as environment variables prior to
826
+ the execution of a step.
827
+
828
+
829
+ Parameters
830
+ ----------
831
+ sources : List[Union[str, Dict[str, Any]]], default: []
832
+ List of secret specs, defining how the secrets are to be retrieved
833
+ role : str, optional, default: None
834
+ Role to use for fetching secrets
835
+ """
836
+ ...
837
+
838
+ @typing.overload
839
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
840
+ ...
841
+
842
+ @typing.overload
843
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
844
+ ...
845
+
846
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
847
+ """
848
+ Specifies secrets to be retrieved and injected as environment variables prior to
849
+ the execution of a step.
850
+
851
+
852
+ Parameters
853
+ ----------
854
+ sources : List[Union[str, Dict[str, Any]]], default: []
855
+ List of secret specs, defining how the secrets are to be retrieved
856
+ role : str, optional, default: None
857
+ Role to use for fetching secrets
858
+ """
859
+ ...
860
+
861
+ @typing.overload
862
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
863
+ """
864
+ Specifies the number of times the task corresponding
865
+ to a step needs to be retried.
866
+
867
+ This decorator is useful for handling transient errors, such as networking issues.
868
+ If your task contains operations that can't be retried safely, e.g. database updates,
869
+ it is advisable to annotate it with `@retry(times=0)`.
870
+
871
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
872
+ decorator will execute a no-op task after all retries have been exhausted,
873
+ ensuring that the flow execution can continue.
874
+
875
+
876
+ Parameters
877
+ ----------
878
+ times : int, default 3
879
+ Number of times to retry this task.
880
+ minutes_between_retries : int, default 2
881
+ Number of minutes between retries.
882
+ """
883
+ ...
884
+
885
+ @typing.overload
886
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
887
+ ...
888
+
889
+ @typing.overload
890
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
891
+ ...
892
+
893
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
894
+ """
895
+ Specifies the number of times the task corresponding
896
+ to a step needs to be retried.
897
+
898
+ This decorator is useful for handling transient errors, such as networking issues.
899
+ If your task contains operations that can't be retried safely, e.g. database updates,
900
+ it is advisable to annotate it with `@retry(times=0)`.
901
+
902
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
903
+ decorator will execute a no-op task after all retries have been exhausted,
904
+ ensuring that the flow execution can continue.
905
+
906
+
907
+ Parameters
908
+ ----------
909
+ times : int, default 3
910
+ Number of times to retry this task.
911
+ minutes_between_retries : int, default 2
912
+ Number of minutes between retries.
792
913
  """
793
914
  ...
794
915
 
@@ -904,61 +1025,127 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope:
904
1025
  - It promotes cache reuse across flows.
905
1026
  - The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
906
1027
 
907
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
908
- The list of repos (models/datasets) to load.
1028
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
1029
+ The list of repos (models/datasets) to load.
1030
+
1031
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
1032
+
1033
+ - If repo (model/dataset) is not found in the datastore:
1034
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
1035
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
1036
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
1037
+
1038
+ - If repo is found in the datastore:
1039
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
1040
+ """
1041
+ ...
1042
+
1043
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1044
+ """
1045
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
1046
+
1047
+ User code call
1048
+ --------------
1049
+ @ollama(
1050
+ models=[...],
1051
+ ...
1052
+ )
1053
+
1054
+ Valid backend options
1055
+ ---------------------
1056
+ - 'local': Run as a separate process on the local task machine.
1057
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
1058
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
909
1059
 
910
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
1060
+ Valid model options
1061
+ -------------------
1062
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
911
1063
 
912
- - If repo (model/dataset) is not found in the datastore:
913
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
914
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
915
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
916
1064
 
917
- - If repo is found in the datastore:
918
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
1065
+ Parameters
1066
+ ----------
1067
+ models: list[str]
1068
+ List of Ollama containers running models in sidecars.
1069
+ backend: str
1070
+ Determines where and how to run the Ollama process.
1071
+ force_pull: bool
1072
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
1073
+ cache_update_policy: str
1074
+ Cache update policy: "auto", "force", or "never".
1075
+ force_cache_update: bool
1076
+ Simple override for "force" cache update policy.
1077
+ debug: bool
1078
+ Whether to turn on verbose debugging logs.
1079
+ circuit_breaker_config: dict
1080
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
1081
+ timeout_config: dict
1082
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
919
1083
  """
920
1084
  ...
921
1085
 
922
1086
  @typing.overload
923
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1087
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
924
1088
  """
925
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
926
- It exists to make it easier for users to know that this decorator should only be used with
927
- a Neo Cloud like CoreWeave.
1089
+ Internal decorator to support Fast bakery
928
1090
  """
929
1091
  ...
930
1092
 
931
1093
  @typing.overload
932
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1094
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
933
1095
  ...
934
1096
 
935
- def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1097
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
936
1098
  """
937
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
938
- It exists to make it easier for users to know that this decorator should only be used with
939
- a Neo Cloud like CoreWeave.
1099
+ Internal decorator to support Fast bakery
940
1100
  """
941
1101
  ...
942
1102
 
943
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1103
+ @typing.overload
1104
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
944
1105
  """
945
- S3 Proxy decorator for routing S3 requests through a local proxy service.
1106
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1107
+
1108
+ Note that you may add multiple `@card` decorators in a step with different parameters.
946
1109
 
947
1110
 
948
1111
  Parameters
949
1112
  ----------
950
- integration_name : str, optional
951
- Name of the S3 proxy integration. If not specified, will use the only
952
- available S3 proxy integration in the namespace (fails if multiple exist).
953
- write_mode : str, optional
954
- The desired behavior during write operations to target (origin) S3 bucket.
955
- allowed options are:
956
- "origin-and-cache" -> write to both the target S3 bucket and local object
957
- storage
958
- "origin" -> only write to the target S3 bucket
959
- "cache" -> only write to the object storage service used for caching
960
- debug : bool, optional
961
- Enable debug logging for proxy operations.
1113
+ type : str, default 'default'
1114
+ Card type.
1115
+ id : str, optional, default None
1116
+ If multiple cards are present, use this id to identify this card.
1117
+ options : Dict[str, Any], default {}
1118
+ Options passed to the card. The contents depend on the card type.
1119
+ timeout : int, default 45
1120
+ Interrupt reporting if it takes more than this many seconds.
1121
+ """
1122
+ ...
1123
+
1124
+ @typing.overload
1125
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1126
+ ...
1127
+
1128
+ @typing.overload
1129
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1130
+ ...
1131
+
1132
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1133
+ """
1134
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1135
+
1136
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1137
+
1138
+
1139
+ Parameters
1140
+ ----------
1141
+ type : str, default 'default'
1142
+ Card type.
1143
+ id : str, optional, default None
1144
+ If multiple cards are present, use this id to identify this card.
1145
+ options : Dict[str, Any], default {}
1146
+ Options passed to the card. The contents depend on the card type.
1147
+ timeout : int, default 45
1148
+ Interrupt reporting if it takes more than this many seconds.
962
1149
  """
963
1150
  ...
964
1151
 
@@ -1052,346 +1239,294 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
1052
1239
  ...
1053
1240
 
1054
1241
  @typing.overload
1055
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1242
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1056
1243
  """
1057
- Specifies a timeout for your step.
1058
-
1059
- This decorator is useful if this step may hang indefinitely.
1060
-
1061
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1062
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1063
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1064
-
1065
- Note that all the values specified in parameters are added together so if you specify
1066
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1067
-
1068
-
1069
- Parameters
1070
- ----------
1071
- seconds : int, default 0
1072
- Number of seconds to wait prior to timing out.
1073
- minutes : int, default 0
1074
- Number of minutes to wait prior to timing out.
1075
- hours : int, default 0
1076
- Number of hours to wait prior to timing out.
1244
+ A simple decorator that demonstrates using CardDecoratorInjector
1245
+ to inject a card and render simple markdown content.
1077
1246
  """
1078
1247
  ...
1079
1248
 
1080
1249
  @typing.overload
1081
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1082
- ...
1083
-
1084
- @typing.overload
1085
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1250
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1086
1251
  ...
1087
1252
 
1088
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1253
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1089
1254
  """
1090
- Specifies a timeout for your step.
1091
-
1092
- This decorator is useful if this step may hang indefinitely.
1093
-
1094
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1095
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1096
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1097
-
1098
- Note that all the values specified in parameters are added together so if you specify
1099
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1100
-
1101
-
1102
- Parameters
1103
- ----------
1104
- seconds : int, default 0
1105
- Number of seconds to wait prior to timing out.
1106
- minutes : int, default 0
1107
- Number of minutes to wait prior to timing out.
1108
- hours : int, default 0
1109
- Number of hours to wait prior to timing out.
1255
+ A simple decorator that demonstrates using CardDecoratorInjector
1256
+ to inject a card and render simple markdown content.
1110
1257
  """
1111
1258
  ...
1112
1259
 
1113
- @typing.overload
1114
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1260
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1115
1261
  """
1116
- Creates a human-readable report, a Metaflow Card, after this step completes.
1117
-
1118
- Note that you may add multiple `@card` decorators in a step with different parameters.
1262
+ Specifies that this step should execute on DGX cloud.
1119
1263
 
1120
1264
 
1121
1265
  Parameters
1122
1266
  ----------
1123
- type : str, default 'default'
1124
- Card type.
1125
- id : str, optional, default None
1126
- If multiple cards are present, use this id to identify this card.
1127
- options : Dict[str, Any], default {}
1128
- Options passed to the card. The contents depend on the card type.
1129
- timeout : int, default 45
1130
- Interrupt reporting if it takes more than this many seconds.
1267
+ gpu : int
1268
+ Number of GPUs to use.
1269
+ gpu_type : str
1270
+ Type of Nvidia GPU to use.
1271
+ queue_timeout : int
1272
+ Time to keep the job in NVCF's queue.
1131
1273
  """
1132
1274
  ...
1133
1275
 
1134
1276
  @typing.overload
1135
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1277
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1278
+ """
1279
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1280
+ It exists to make it easier for users to know that this decorator should only be used with
1281
+ a Neo Cloud like Nebius.
1282
+ """
1136
1283
  ...
1137
1284
 
1138
1285
  @typing.overload
1139
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1140
- ...
1141
-
1142
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1143
- """
1144
- Creates a human-readable report, a Metaflow Card, after this step completes.
1145
-
1146
- Note that you may add multiple `@card` decorators in a step with different parameters.
1147
-
1148
-
1149
- Parameters
1150
- ----------
1151
- type : str, default 'default'
1152
- Card type.
1153
- id : str, optional, default None
1154
- If multiple cards are present, use this id to identify this card.
1155
- options : Dict[str, Any], default {}
1156
- Options passed to the card. The contents depend on the card type.
1157
- timeout : int, default 45
1158
- Interrupt reporting if it takes more than this many seconds.
1159
- """
1286
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1160
1287
  ...
1161
1288
 
1162
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1289
+ def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1163
1290
  """
1164
- Specifies that this step should execute on DGX cloud.
1165
-
1166
-
1167
- Parameters
1168
- ----------
1169
- gpu : int
1170
- Number of GPUs to use.
1171
- gpu_type : str
1172
- Type of Nvidia GPU to use.
1173
- queue_timeout : int
1174
- Time to keep the job in NVCF's queue.
1291
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1292
+ It exists to make it easier for users to know that this decorator should only be used with
1293
+ a Neo Cloud like Nebius.
1175
1294
  """
1176
1295
  ...
1177
1296
 
1178
1297
  @typing.overload
1179
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1180
- """
1181
- Enables loading / saving of models within a step.
1182
-
1183
- > Examples
1184
- - Saving Models
1185
- ```python
1186
- @model
1187
- @step
1188
- def train(self):
1189
- # current.model.save returns a dictionary reference to the model saved
1190
- self.my_model = current.model.save(
1191
- path_to_my_model,
1192
- label="my_model",
1193
- metadata={
1194
- "epochs": 10,
1195
- "batch-size": 32,
1196
- "learning-rate": 0.001,
1197
- }
1198
- )
1199
- self.next(self.test)
1200
-
1201
- @model(load="my_model")
1202
- @step
1203
- def test(self):
1204
- # `current.model.loaded` returns a dictionary of the loaded models
1205
- # where the key is the name of the artifact and the value is the path to the model
1206
- print(os.listdir(current.model.loaded["my_model"]))
1207
- self.next(self.end)
1208
- ```
1298
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1299
+ """
1300
+ Specifies the PyPI packages for the step.
1209
1301
 
1210
- - Loading models
1211
- ```python
1212
- @step
1213
- def train(self):
1214
- # current.model.load returns the path to the model loaded
1215
- checkpoint_path = current.model.load(
1216
- self.checkpoint_key,
1217
- )
1218
- model_path = current.model.load(
1219
- self.model,
1220
- )
1221
- self.next(self.test)
1222
- ```
1302
+ Information in this decorator will augment any
1303
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1304
+ you can use `@pypi_base` to set packages required by all
1305
+ steps and use `@pypi` to specify step-specific overrides.
1223
1306
 
1224
1307
 
1225
1308
  Parameters
1226
1309
  ----------
1227
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1228
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1229
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1230
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1231
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1232
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1233
-
1234
- temp_dir_root : str, default: None
1235
- The root directory under which `current.model.loaded` will store loaded models
1310
+ packages : Dict[str, str], default: {}
1311
+ Packages to use for this step. The key is the name of the package
1312
+ and the value is the version to use.
1313
+ python : str, optional, default: None
1314
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1315
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1236
1316
  """
1237
1317
  ...
1238
1318
 
1239
1319
  @typing.overload
1240
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1320
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1241
1321
  ...
1242
1322
 
1243
1323
  @typing.overload
1244
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1324
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1245
1325
  ...
1246
1326
 
1247
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1327
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1248
1328
  """
1249
- Enables loading / saving of models within a step.
1250
-
1251
- > Examples
1252
- - Saving Models
1253
- ```python
1254
- @model
1255
- @step
1256
- def train(self):
1257
- # current.model.save returns a dictionary reference to the model saved
1258
- self.my_model = current.model.save(
1259
- path_to_my_model,
1260
- label="my_model",
1261
- metadata={
1262
- "epochs": 10,
1263
- "batch-size": 32,
1264
- "learning-rate": 0.001,
1265
- }
1266
- )
1267
- self.next(self.test)
1268
-
1269
- @model(load="my_model")
1270
- @step
1271
- def test(self):
1272
- # `current.model.loaded` returns a dictionary of the loaded models
1273
- # where the key is the name of the artifact and the value is the path to the model
1274
- print(os.listdir(current.model.loaded["my_model"]))
1275
- self.next(self.end)
1276
- ```
1329
+ Specifies the PyPI packages for the step.
1277
1330
 
1278
- - Loading models
1279
- ```python
1280
- @step
1281
- def train(self):
1282
- # current.model.load returns the path to the model loaded
1283
- checkpoint_path = current.model.load(
1284
- self.checkpoint_key,
1285
- )
1286
- model_path = current.model.load(
1287
- self.model,
1288
- )
1289
- self.next(self.test)
1290
- ```
1331
+ Information in this decorator will augment any
1332
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1333
+ you can use `@pypi_base` to set packages required by all
1334
+ steps and use `@pypi` to specify step-specific overrides.
1291
1335
 
1292
1336
 
1293
1337
  Parameters
1294
1338
  ----------
1295
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1296
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1297
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1298
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1299
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1300
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1301
-
1302
- temp_dir_root : str, default: None
1303
- The root directory under which `current.model.loaded` will store loaded models
1339
+ packages : Dict[str, str], default: {}
1340
+ Packages to use for this step. The key is the name of the package
1341
+ and the value is the version to use.
1342
+ python : str, optional, default: None
1343
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1344
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1304
1345
  """
1305
1346
  ...
1306
1347
 
1307
- @typing.overload
1308
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1348
+ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1309
1349
  """
1310
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1311
- It exists to make it easier for users to know that this decorator should only be used with
1312
- a Neo Cloud like Nebius.
1350
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
1351
+
1352
+ User code call
1353
+ --------------
1354
+ @vllm(
1355
+ model="...",
1356
+ ...
1357
+ )
1358
+
1359
+ Valid backend options
1360
+ ---------------------
1361
+ - 'local': Run as a separate process on the local task machine.
1362
+
1363
+ Valid model options
1364
+ -------------------
1365
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
1366
+
1367
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
1368
+ If you need multiple models, you must create multiple @vllm decorators.
1369
+
1370
+
1371
+ Parameters
1372
+ ----------
1373
+ model: str
1374
+ HuggingFace model identifier to be served by vLLM.
1375
+ backend: str
1376
+ Determines where and how to run the vLLM process.
1377
+ openai_api_server: bool
1378
+ Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
1379
+ Default is False (uses native engine).
1380
+ Set to True for backward compatibility with existing code.
1381
+ debug: bool
1382
+ Whether to turn on verbose debugging logs.
1383
+ card_refresh_interval: int
1384
+ Interval in seconds for refreshing the vLLM status card.
1385
+ Only used when openai_api_server=True.
1386
+ max_retries: int
1387
+ Maximum number of retries checking for vLLM server startup.
1388
+ Only used when openai_api_server=True.
1389
+ retry_alert_frequency: int
1390
+ Frequency of alert logs for vLLM server startup retries.
1391
+ Only used when openai_api_server=True.
1392
+ engine_args : dict
1393
+ Additional keyword arguments to pass to the vLLM engine.
1394
+ For example, `tensor_parallel_size=2`.
1313
1395
  """
1314
1396
  ...
1315
1397
 
1316
- @typing.overload
1317
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1318
- ...
1319
-
1320
- def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1398
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1321
1399
  """
1322
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1323
- It exists to make it easier for users to know that this decorator should only be used with
1324
- a Neo Cloud like Nebius.
1400
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1401
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1402
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1403
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1404
+ starts only after all sensors finish.
1405
+
1406
+
1407
+ Parameters
1408
+ ----------
1409
+ timeout : int
1410
+ Time, in seconds before the task times out and fails. (Default: 3600)
1411
+ poke_interval : int
1412
+ Time in seconds that the job should wait in between each try. (Default: 60)
1413
+ mode : str
1414
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1415
+ exponential_backoff : bool
1416
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1417
+ pool : str
1418
+ the slot pool this task should run in,
1419
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1420
+ soft_fail : bool
1421
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1422
+ name : str
1423
+ Name of the sensor on Airflow
1424
+ description : str
1425
+ Description of sensor in the Airflow UI
1426
+ bucket_key : Union[str, List[str]]
1427
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1428
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1429
+ bucket_name : str
1430
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1431
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1432
+ wildcard_match : bool
1433
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1434
+ aws_conn_id : str
1435
+ a reference to the s3 connection on Airflow. (Default: None)
1436
+ verify : bool
1437
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1325
1438
  """
1326
1439
  ...
1327
1440
 
1328
1441
  @typing.overload
1329
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1442
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1330
1443
  """
1331
- Decorator prototype for all step decorators. This function gets specialized
1332
- and imported for all decorators types by _import_plugin_decorators().
1444
+ Specifies the times when the flow should be run when running on a
1445
+ production scheduler.
1446
+
1447
+
1448
+ Parameters
1449
+ ----------
1450
+ hourly : bool, default False
1451
+ Run the workflow hourly.
1452
+ daily : bool, default True
1453
+ Run the workflow daily.
1454
+ weekly : bool, default False
1455
+ Run the workflow weekly.
1456
+ cron : str, optional, default None
1457
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1458
+ specified by this expression.
1459
+ timezone : str, optional, default None
1460
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1461
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1333
1462
  """
1334
1463
  ...
1335
1464
 
1336
1465
  @typing.overload
1337
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1466
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1338
1467
  ...
1339
1468
 
1340
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1469
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1341
1470
  """
1342
- Decorator prototype for all step decorators. This function gets specialized
1343
- and imported for all decorators types by _import_plugin_decorators().
1471
+ Specifies the times when the flow should be run when running on a
1472
+ production scheduler.
1473
+
1474
+
1475
+ Parameters
1476
+ ----------
1477
+ hourly : bool, default False
1478
+ Run the workflow hourly.
1479
+ daily : bool, default True
1480
+ Run the workflow daily.
1481
+ weekly : bool, default False
1482
+ Run the workflow weekly.
1483
+ cron : str, optional, default None
1484
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1485
+ specified by this expression.
1486
+ timezone : str, optional, default None
1487
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1488
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1344
1489
  """
1345
1490
  ...
1346
1491
 
1347
1492
  @typing.overload
1348
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1493
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1349
1494
  """
1350
- Specifies that the step will success under all circumstances.
1351
-
1352
- The decorator will create an optional artifact, specified by `var`, which
1353
- contains the exception raised. You can use it to detect the presence
1354
- of errors, indicating that all happy-path artifacts produced by the step
1355
- are missing.
1495
+ Specifies the PyPI packages for all steps of the flow.
1356
1496
 
1497
+ Use `@pypi_base` to set common packages required by all
1498
+ steps and use `@pypi` to specify step-specific overrides.
1357
1499
 
1358
1500
  Parameters
1359
1501
  ----------
1360
- var : str, optional, default None
1361
- Name of the artifact in which to store the caught exception.
1362
- If not specified, the exception is not stored.
1363
- print_exception : bool, default True
1364
- Determines whether or not the exception is printed to
1365
- stdout when caught.
1502
+ packages : Dict[str, str], default: {}
1503
+ Packages to use for this flow. The key is the name of the package
1504
+ and the value is the version to use.
1505
+ python : str, optional, default: None
1506
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1507
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1366
1508
  """
1367
1509
  ...
1368
1510
 
1369
1511
  @typing.overload
1370
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1371
- ...
1372
-
1373
- @typing.overload
1374
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1512
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1375
1513
  ...
1376
1514
 
1377
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1515
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1378
1516
  """
1379
- Specifies that the step will success under all circumstances.
1380
-
1381
- The decorator will create an optional artifact, specified by `var`, which
1382
- contains the exception raised. You can use it to detect the presence
1383
- of errors, indicating that all happy-path artifacts produced by the step
1384
- are missing.
1517
+ Specifies the PyPI packages for all steps of the flow.
1385
1518
 
1519
+ Use `@pypi_base` to set common packages required by all
1520
+ steps and use `@pypi` to specify step-specific overrides.
1386
1521
 
1387
1522
  Parameters
1388
1523
  ----------
1389
- var : str, optional, default None
1390
- Name of the artifact in which to store the caught exception.
1391
- If not specified, the exception is not stored.
1392
- print_exception : bool, default True
1393
- Determines whether or not the exception is printed to
1394
- stdout when caught.
1524
+ packages : Dict[str, str], default: {}
1525
+ Packages to use for this flow. The key is the name of the package
1526
+ and the value is the version to use.
1527
+ python : str, optional, default: None
1528
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1529
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1395
1530
  """
1396
1531
  ...
1397
1532
 
@@ -1484,243 +1619,58 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1484
1619
  - `test.my_experiment`
1485
1620
  - `prod.staging`
1486
1621
 
1487
-
1488
- Parameters
1489
- ----------
1490
- flow : Union[str, Dict[str, str]], optional, default None
1491
- Upstream flow dependency for this flow.
1492
- flows : List[Union[str, Dict[str, str]]], default []
1493
- Upstream flow dependencies for this flow.
1494
- options : Dict[str, Any], default {}
1495
- Backend-specific configuration for tuning eventing behavior.
1496
- """
1497
- ...
1498
-
1499
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1500
- """
1501
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1502
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1503
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1504
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1505
- starts only after all sensors finish.
1506
-
1507
-
1508
- Parameters
1509
- ----------
1510
- timeout : int
1511
- Time, in seconds before the task times out and fails. (Default: 3600)
1512
- poke_interval : int
1513
- Time in seconds that the job should wait in between each try. (Default: 60)
1514
- mode : str
1515
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1516
- exponential_backoff : bool
1517
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1518
- pool : str
1519
- the slot pool this task should run in,
1520
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1521
- soft_fail : bool
1522
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1523
- name : str
1524
- Name of the sensor on Airflow
1525
- description : str
1526
- Description of sensor in the Airflow UI
1527
- bucket_key : Union[str, List[str]]
1528
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1529
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1530
- bucket_name : str
1531
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1532
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1533
- wildcard_match : bool
1534
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1535
- aws_conn_id : str
1536
- a reference to the s3 connection on Airflow. (Default: None)
1537
- verify : bool
1538
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1539
- """
1540
- ...
1541
-
1542
- @typing.overload
1543
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1544
- """
1545
- Specifies the event(s) that this flow depends on.
1546
-
1547
- ```
1548
- @trigger(event='foo')
1549
- ```
1550
- or
1551
- ```
1552
- @trigger(events=['foo', 'bar'])
1553
- ```
1554
-
1555
- Additionally, you can specify the parameter mappings
1556
- to map event payload to Metaflow parameters for the flow.
1557
- ```
1558
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1559
- ```
1560
- or
1561
- ```
1562
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1563
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1564
- ```
1565
-
1566
- 'parameters' can also be a list of strings and tuples like so:
1567
- ```
1568
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1569
- ```
1570
- This is equivalent to:
1571
- ```
1572
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1573
- ```
1574
-
1575
-
1576
- Parameters
1577
- ----------
1578
- event : Union[str, Dict[str, Any]], optional, default None
1579
- Event dependency for this flow.
1580
- events : List[Union[str, Dict[str, Any]]], default []
1581
- Events dependency for this flow.
1582
- options : Dict[str, Any], default {}
1583
- Backend-specific configuration for tuning eventing behavior.
1584
- """
1585
- ...
1586
-
1587
- @typing.overload
1588
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1589
- ...
1590
-
1591
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1592
- """
1593
- Specifies the event(s) that this flow depends on.
1594
-
1595
- ```
1596
- @trigger(event='foo')
1597
- ```
1598
- or
1599
- ```
1600
- @trigger(events=['foo', 'bar'])
1601
- ```
1602
-
1603
- Additionally, you can specify the parameter mappings
1604
- to map event payload to Metaflow parameters for the flow.
1605
- ```
1606
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1607
- ```
1608
- or
1609
- ```
1610
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1611
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1612
- ```
1613
-
1614
- 'parameters' can also be a list of strings and tuples like so:
1615
- ```
1616
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1617
- ```
1618
- This is equivalent to:
1619
- ```
1620
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1621
- ```
1622
-
1623
-
1624
- Parameters
1625
- ----------
1626
- event : Union[str, Dict[str, Any]], optional, default None
1627
- Event dependency for this flow.
1628
- events : List[Union[str, Dict[str, Any]]], default []
1629
- Events dependency for this flow.
1630
- options : Dict[str, Any], default {}
1631
- Backend-specific configuration for tuning eventing behavior.
1632
- """
1633
- ...
1634
-
1635
- @typing.overload
1636
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1637
- """
1638
- Specifies the Conda environment for all steps of the flow.
1639
-
1640
- Use `@conda_base` to set common libraries required by all
1641
- steps and use `@conda` to specify step-specific additions.
1642
-
1643
-
1644
- Parameters
1645
- ----------
1646
- packages : Dict[str, str], default {}
1647
- Packages to use for this flow. The key is the name of the package
1648
- and the value is the version to use.
1649
- libraries : Dict[str, str], default {}
1650
- Supported for backward compatibility. When used with packages, packages will take precedence.
1651
- python : str, optional, default None
1652
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1653
- that the version used will correspond to the version of the Python interpreter used to start the run.
1654
- disabled : bool, default False
1655
- If set to True, disables Conda.
1656
- """
1657
- ...
1658
-
1659
- @typing.overload
1660
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1661
- ...
1662
-
1663
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1664
- """
1665
- Specifies the Conda environment for all steps of the flow.
1666
-
1667
- Use `@conda_base` to set common libraries required by all
1668
- steps and use `@conda` to specify step-specific additions.
1669
-
1670
-
1671
- Parameters
1672
- ----------
1673
- packages : Dict[str, str], default {}
1674
- Packages to use for this flow. The key is the name of the package
1675
- and the value is the version to use.
1676
- libraries : Dict[str, str], default {}
1677
- Supported for backward compatibility. When used with packages, packages will take precedence.
1678
- python : str, optional, default None
1679
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1680
- that the version used will correspond to the version of the Python interpreter used to start the run.
1681
- disabled : bool, default False
1682
- If set to True, disables Conda.
1683
- """
1684
- ...
1685
-
1686
- @typing.overload
1687
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1688
- """
1689
- Specifies the PyPI packages for all steps of the flow.
1690
-
1691
- Use `@pypi_base` to set common packages required by all
1692
- steps and use `@pypi` to specify step-specific overrides.
1693
-
1694
- Parameters
1695
- ----------
1696
- packages : Dict[str, str], default: {}
1697
- Packages to use for this flow. The key is the name of the package
1698
- and the value is the version to use.
1699
- python : str, optional, default: None
1700
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1701
- that the version used will correspond to the version of the Python interpreter used to start the run.
1622
+
1623
+ Parameters
1624
+ ----------
1625
+ flow : Union[str, Dict[str, str]], optional, default None
1626
+ Upstream flow dependency for this flow.
1627
+ flows : List[Union[str, Dict[str, str]]], default []
1628
+ Upstream flow dependencies for this flow.
1629
+ options : Dict[str, Any], default {}
1630
+ Backend-specific configuration for tuning eventing behavior.
1702
1631
  """
1703
1632
  ...
1704
1633
 
1705
- @typing.overload
1706
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1707
- ...
1708
-
1709
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1634
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1710
1635
  """
1711
- Specifies the PyPI packages for all steps of the flow.
1636
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1637
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1712
1638
 
1713
- Use `@pypi_base` to set common packages required by all
1714
- steps and use `@pypi` to specify step-specific overrides.
1715
1639
 
1716
1640
  Parameters
1717
1641
  ----------
1718
- packages : Dict[str, str], default: {}
1719
- Packages to use for this flow. The key is the name of the package
1720
- and the value is the version to use.
1721
- python : str, optional, default: None
1722
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1723
- that the version used will correspond to the version of the Python interpreter used to start the run.
1642
+ timeout : int
1643
+ Time, in seconds before the task times out and fails. (Default: 3600)
1644
+ poke_interval : int
1645
+ Time in seconds that the job should wait in between each try. (Default: 60)
1646
+ mode : str
1647
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1648
+ exponential_backoff : bool
1649
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1650
+ pool : str
1651
+ the slot pool this task should run in,
1652
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1653
+ soft_fail : bool
1654
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1655
+ name : str
1656
+ Name of the sensor on Airflow
1657
+ description : str
1658
+ Description of sensor in the Airflow UI
1659
+ external_dag_id : str
1660
+ The dag_id that contains the task you want to wait for.
1661
+ external_task_ids : List[str]
1662
+ The list of task_ids that you want to wait for.
1663
+ If None (default value) the sensor waits for the DAG. (Default: None)
1664
+ allowed_states : List[str]
1665
+ Iterable of allowed states, (Default: ['success'])
1666
+ failed_states : List[str]
1667
+ Iterable of failed or dis-allowed states. (Default: None)
1668
+ execution_delta : datetime.timedelta
1669
+ time difference with the previous execution to look at,
1670
+ the default is the same logical date as the current task or DAG. (Default: None)
1671
+ check_existence: bool
1672
+ Set to True to check if the external task exists or check if
1673
+ the DAG to wait for exists. (Default: True)
1724
1674
  """
1725
1675
  ...
1726
1676
 
@@ -1838,49 +1788,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1838
1788
  """
1839
1789
  ...
1840
1790
 
1841
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1842
- """
1843
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1844
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1845
-
1846
-
1847
- Parameters
1848
- ----------
1849
- timeout : int
1850
- Time, in seconds before the task times out and fails. (Default: 3600)
1851
- poke_interval : int
1852
- Time in seconds that the job should wait in between each try. (Default: 60)
1853
- mode : str
1854
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1855
- exponential_backoff : bool
1856
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1857
- pool : str
1858
- the slot pool this task should run in,
1859
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1860
- soft_fail : bool
1861
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1862
- name : str
1863
- Name of the sensor on Airflow
1864
- description : str
1865
- Description of sensor in the Airflow UI
1866
- external_dag_id : str
1867
- The dag_id that contains the task you want to wait for.
1868
- external_task_ids : List[str]
1869
- The list of task_ids that you want to wait for.
1870
- If None (default value) the sensor waits for the DAG. (Default: None)
1871
- allowed_states : List[str]
1872
- Iterable of allowed states, (Default: ['success'])
1873
- failed_states : List[str]
1874
- Iterable of failed or dis-allowed states. (Default: None)
1875
- execution_delta : datetime.timedelta
1876
- time difference with the previous execution to look at,
1877
- the default is the same logical date as the current task or DAG. (Default: None)
1878
- check_existence: bool
1879
- Set to True to check if the external task exists or check if
1880
- the DAG to wait for exists. (Default: True)
1881
- """
1882
- ...
1883
-
1884
1791
  def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1885
1792
  """
1886
1793
  Specifies what flows belong to the same project.
@@ -1917,53 +1824,146 @@ def project(*, name: str, branch: typing.Optional[str] = None, production: bool
1917
1824
  ...
1918
1825
 
1919
1826
  @typing.overload
1920
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1827
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1921
1828
  """
1922
- Specifies the times when the flow should be run when running on a
1923
- production scheduler.
1829
+ Specifies the Conda environment for all steps of the flow.
1830
+
1831
+ Use `@conda_base` to set common libraries required by all
1832
+ steps and use `@conda` to specify step-specific additions.
1924
1833
 
1925
1834
 
1926
1835
  Parameters
1927
1836
  ----------
1928
- hourly : bool, default False
1929
- Run the workflow hourly.
1930
- daily : bool, default True
1931
- Run the workflow daily.
1932
- weekly : bool, default False
1933
- Run the workflow weekly.
1934
- cron : str, optional, default None
1935
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1936
- specified by this expression.
1937
- timezone : str, optional, default None
1938
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1939
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1837
+ packages : Dict[str, str], default {}
1838
+ Packages to use for this flow. The key is the name of the package
1839
+ and the value is the version to use.
1840
+ libraries : Dict[str, str], default {}
1841
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1842
+ python : str, optional, default None
1843
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1844
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1845
+ disabled : bool, default False
1846
+ If set to True, disables Conda.
1940
1847
  """
1941
1848
  ...
1942
1849
 
1943
1850
  @typing.overload
1944
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1851
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1945
1852
  ...
1946
1853
 
1947
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1854
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1948
1855
  """
1949
- Specifies the times when the flow should be run when running on a
1950
- production scheduler.
1856
+ Specifies the Conda environment for all steps of the flow.
1857
+
1858
+ Use `@conda_base` to set common libraries required by all
1859
+ steps and use `@conda` to specify step-specific additions.
1951
1860
 
1952
1861
 
1953
1862
  Parameters
1954
1863
  ----------
1955
- hourly : bool, default False
1956
- Run the workflow hourly.
1957
- daily : bool, default True
1958
- Run the workflow daily.
1959
- weekly : bool, default False
1960
- Run the workflow weekly.
1961
- cron : str, optional, default None
1962
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1963
- specified by this expression.
1964
- timezone : str, optional, default None
1965
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1966
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1864
+ packages : Dict[str, str], default {}
1865
+ Packages to use for this flow. The key is the name of the package
1866
+ and the value is the version to use.
1867
+ libraries : Dict[str, str], default {}
1868
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1869
+ python : str, optional, default None
1870
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1871
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1872
+ disabled : bool, default False
1873
+ If set to True, disables Conda.
1874
+ """
1875
+ ...
1876
+
1877
+ @typing.overload
1878
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1879
+ """
1880
+ Specifies the event(s) that this flow depends on.
1881
+
1882
+ ```
1883
+ @trigger(event='foo')
1884
+ ```
1885
+ or
1886
+ ```
1887
+ @trigger(events=['foo', 'bar'])
1888
+ ```
1889
+
1890
+ Additionally, you can specify the parameter mappings
1891
+ to map event payload to Metaflow parameters for the flow.
1892
+ ```
1893
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1894
+ ```
1895
+ or
1896
+ ```
1897
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1898
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1899
+ ```
1900
+
1901
+ 'parameters' can also be a list of strings and tuples like so:
1902
+ ```
1903
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1904
+ ```
1905
+ This is equivalent to:
1906
+ ```
1907
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1908
+ ```
1909
+
1910
+
1911
+ Parameters
1912
+ ----------
1913
+ event : Union[str, Dict[str, Any]], optional, default None
1914
+ Event dependency for this flow.
1915
+ events : List[Union[str, Dict[str, Any]]], default []
1916
+ Events dependency for this flow.
1917
+ options : Dict[str, Any], default {}
1918
+ Backend-specific configuration for tuning eventing behavior.
1919
+ """
1920
+ ...
1921
+
1922
+ @typing.overload
1923
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1924
+ ...
1925
+
1926
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1927
+ """
1928
+ Specifies the event(s) that this flow depends on.
1929
+
1930
+ ```
1931
+ @trigger(event='foo')
1932
+ ```
1933
+ or
1934
+ ```
1935
+ @trigger(events=['foo', 'bar'])
1936
+ ```
1937
+
1938
+ Additionally, you can specify the parameter mappings
1939
+ to map event payload to Metaflow parameters for the flow.
1940
+ ```
1941
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1942
+ ```
1943
+ or
1944
+ ```
1945
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1946
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1947
+ ```
1948
+
1949
+ 'parameters' can also be a list of strings and tuples like so:
1950
+ ```
1951
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1952
+ ```
1953
+ This is equivalent to:
1954
+ ```
1955
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1956
+ ```
1957
+
1958
+
1959
+ Parameters
1960
+ ----------
1961
+ event : Union[str, Dict[str, Any]], optional, default None
1962
+ Event dependency for this flow.
1963
+ events : List[Union[str, Dict[str, Any]]], default []
1964
+ Events dependency for this flow.
1965
+ options : Dict[str, Any], default {}
1966
+ Backend-specific configuration for tuning eventing behavior.
1967
1967
  """
1968
1968
  ...
1969
1969