ob-metaflow-stubs 6.0.9.2__py2.py3-none-any.whl → 6.0.9.3__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow-stubs might be problematic. Click here for more details.

Files changed (262) hide show
  1. metaflow-stubs/__init__.pyi +812 -812
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +4 -4
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +1 -1
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +3 -3
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/meta_files.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +1 -1
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +53 -53
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +1 -1
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +1 -1
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +1 -1
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +1 -1
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +1 -1
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +1 -1
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +1 -1
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +1 -1
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +3 -3
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +1 -1
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +1 -1
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +1 -1
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +1 -1
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +2 -2
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +1 -1
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +1 -1
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +4 -4
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +1 -1
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +1 -1
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +3 -3
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +1 -1
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +1 -1
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +2 -2
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +1 -1
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  110. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  111. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  112. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  113. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  114. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +1 -1
  116. metaflow-stubs/multicore_utils.pyi +1 -1
  117. metaflow-stubs/ob_internal.pyi +1 -1
  118. metaflow-stubs/packaging_sys/__init__.pyi +5 -5
  119. metaflow-stubs/packaging_sys/backend.pyi +2 -2
  120. metaflow-stubs/packaging_sys/distribution_support.pyi +3 -3
  121. metaflow-stubs/packaging_sys/tar_backend.pyi +3 -3
  122. metaflow-stubs/packaging_sys/utils.pyi +1 -1
  123. metaflow-stubs/packaging_sys/v1.pyi +2 -2
  124. metaflow-stubs/parameters.pyi +2 -2
  125. metaflow-stubs/plugins/__init__.pyi +12 -12
  126. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  127. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  128. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  129. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  130. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  131. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  132. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  133. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  134. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  135. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  136. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  137. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  138. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  139. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +1 -1
  140. metaflow-stubs/plugins/argo/exit_hooks.pyi +1 -1
  141. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  142. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  143. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  144. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  145. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  146. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  147. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  148. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  149. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  150. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  151. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  152. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  153. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  154. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  155. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +1 -1
  157. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  158. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  159. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  160. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  161. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  162. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  163. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  164. metaflow-stubs/plugins/cards/__init__.pyi +5 -5
  165. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  166. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  167. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  168. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  169. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  170. metaflow-stubs/plugins/cards/card_modules/basic.pyi +1 -1
  171. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  172. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  173. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  174. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  175. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  176. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  177. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  178. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  179. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  180. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  181. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  182. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  183. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  184. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  185. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  186. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  187. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  188. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  189. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  190. metaflow-stubs/plugins/exit_hook/__init__.pyi +1 -1
  191. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +1 -1
  192. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  193. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  194. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  195. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  196. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  197. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  198. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  199. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  200. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  201. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
  202. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  203. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  204. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  205. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  206. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  207. metaflow-stubs/plugins/ollama/__init__.pyi +1 -1
  208. metaflow-stubs/plugins/optuna/__init__.pyi +1 -1
  209. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  210. metaflow-stubs/plugins/perimeters.pyi +1 -1
  211. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  212. metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
  213. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  214. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  215. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  216. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  217. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  218. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  219. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  220. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  221. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  222. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
  223. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  224. metaflow-stubs/plugins/secrets/secrets_func.pyi +1 -1
  225. metaflow-stubs/plugins/secrets/secrets_spec.pyi +1 -1
  226. metaflow-stubs/plugins/secrets/utils.pyi +1 -1
  227. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  228. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  229. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
  230. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  231. metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
  232. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  233. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  234. metaflow-stubs/profilers/__init__.pyi +1 -1
  235. metaflow-stubs/pylint_wrapper.pyi +1 -1
  236. metaflow-stubs/runner/__init__.pyi +1 -1
  237. metaflow-stubs/runner/deployer.pyi +4 -4
  238. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  239. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  240. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  241. metaflow-stubs/runner/nbrun.pyi +1 -1
  242. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  243. metaflow-stubs/runner/utils.pyi +3 -3
  244. metaflow-stubs/system/__init__.pyi +1 -1
  245. metaflow-stubs/system/system_logger.pyi +2 -2
  246. metaflow-stubs/system/system_monitor.pyi +1 -1
  247. metaflow-stubs/tagging_util.pyi +1 -1
  248. metaflow-stubs/tuple_util.pyi +1 -1
  249. metaflow-stubs/user_configs/__init__.pyi +1 -1
  250. metaflow-stubs/user_configs/config_options.pyi +1 -1
  251. metaflow-stubs/user_configs/config_parameters.pyi +4 -4
  252. metaflow-stubs/user_decorators/__init__.pyi +1 -1
  253. metaflow-stubs/user_decorators/common.pyi +1 -1
  254. metaflow-stubs/user_decorators/mutable_flow.pyi +3 -3
  255. metaflow-stubs/user_decorators/mutable_step.pyi +4 -4
  256. metaflow-stubs/user_decorators/user_flow_decorator.pyi +3 -3
  257. metaflow-stubs/user_decorators/user_step_decorator.pyi +3 -3
  258. {ob_metaflow_stubs-6.0.9.2.dist-info → ob_metaflow_stubs-6.0.9.3.dist-info}/METADATA +1 -1
  259. ob_metaflow_stubs-6.0.9.3.dist-info/RECORD +262 -0
  260. ob_metaflow_stubs-6.0.9.2.dist-info/RECORD +0 -262
  261. {ob_metaflow_stubs-6.0.9.2.dist-info → ob_metaflow_stubs-6.0.9.3.dist-info}/WHEEL +0 -0
  262. {ob_metaflow_stubs-6.0.9.2.dist-info → ob_metaflow_stubs-6.0.9.3.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.18.1.1+obcheckpoint(0.2.4);ob(v1) #
4
- # Generated on 2025-08-29T18:32:22.531594 #
4
+ # Generated on 2025-09-02T19:19:25.341768 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -40,9 +40,9 @@ from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
42
  from . import metaflow_git as metaflow_git
43
- from . import tuple_util as tuple_util
44
- from . import cards as cards
45
43
  from . import events as events
44
+ from . import cards as cards
45
+ from . import tuple_util as tuple_util
46
46
  from . import runner as runner
47
47
  from . import plugins as plugins
48
48
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
@@ -167,57 +167,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
167
167
  """
168
168
  ...
169
169
 
170
- @typing.overload
171
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
172
- """
173
- Specifies that the step will success under all circumstances.
174
-
175
- The decorator will create an optional artifact, specified by `var`, which
176
- contains the exception raised. You can use it to detect the presence
177
- of errors, indicating that all happy-path artifacts produced by the step
178
- are missing.
179
-
180
-
181
- Parameters
182
- ----------
183
- var : str, optional, default None
184
- Name of the artifact in which to store the caught exception.
185
- If not specified, the exception is not stored.
186
- print_exception : bool, default True
187
- Determines whether or not the exception is printed to
188
- stdout when caught.
189
- """
190
- ...
191
-
192
- @typing.overload
193
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
194
- ...
195
-
196
- @typing.overload
197
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
198
- ...
199
-
200
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
201
- """
202
- Specifies that the step will success under all circumstances.
203
-
204
- The decorator will create an optional artifact, specified by `var`, which
205
- contains the exception raised. You can use it to detect the presence
206
- of errors, indicating that all happy-path artifacts produced by the step
207
- are missing.
208
-
209
-
210
- Parameters
211
- ----------
212
- var : str, optional, default None
213
- Name of the artifact in which to store the caught exception.
214
- If not specified, the exception is not stored.
215
- print_exception : bool, default True
216
- Determines whether or not the exception is printed to
217
- stdout when caught.
218
- """
219
- ...
220
-
221
170
  @typing.overload
222
171
  def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
223
172
  """
@@ -278,234 +227,178 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
278
227
  ...
279
228
 
280
229
  @typing.overload
281
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
230
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
282
231
  """
283
- Enables loading / saving of models within a step.
284
-
285
- > Examples
286
- - Saving Models
287
- ```python
288
- @model
289
- @step
290
- def train(self):
291
- # current.model.save returns a dictionary reference to the model saved
292
- self.my_model = current.model.save(
293
- path_to_my_model,
294
- label="my_model",
295
- metadata={
296
- "epochs": 10,
297
- "batch-size": 32,
298
- "learning-rate": 0.001,
299
- }
300
- )
301
- self.next(self.test)
302
-
303
- @model(load="my_model")
304
- @step
305
- def test(self):
306
- # `current.model.loaded` returns a dictionary of the loaded models
307
- # where the key is the name of the artifact and the value is the path to the model
308
- print(os.listdir(current.model.loaded["my_model"]))
309
- self.next(self.end)
310
- ```
311
-
312
- - Loading models
313
- ```python
314
- @step
315
- def train(self):
316
- # current.model.load returns the path to the model loaded
317
- checkpoint_path = current.model.load(
318
- self.checkpoint_key,
319
- )
320
- model_path = current.model.load(
321
- self.model,
322
- )
323
- self.next(self.test)
324
- ```
325
-
326
-
327
- Parameters
328
- ----------
329
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
330
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
331
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
332
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
333
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
334
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
335
-
336
- temp_dir_root : str, default: None
337
- The root directory under which `current.model.loaded` will store loaded models
232
+ Internal decorator to support Fast bakery
338
233
  """
339
234
  ...
340
235
 
341
236
  @typing.overload
342
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
237
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
343
238
  ...
344
239
 
345
- @typing.overload
346
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
240
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
241
+ """
242
+ Internal decorator to support Fast bakery
243
+ """
347
244
  ...
348
245
 
349
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
246
+ @typing.overload
247
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
350
248
  """
351
- Enables loading / saving of models within a step.
352
-
353
- > Examples
354
- - Saving Models
355
- ```python
356
- @model
357
- @step
358
- def train(self):
359
- # current.model.save returns a dictionary reference to the model saved
360
- self.my_model = current.model.save(
361
- path_to_my_model,
362
- label="my_model",
363
- metadata={
364
- "epochs": 10,
365
- "batch-size": 32,
366
- "learning-rate": 0.001,
367
- }
368
- )
369
- self.next(self.test)
370
-
371
- @model(load="my_model")
372
- @step
373
- def test(self):
374
- # `current.model.loaded` returns a dictionary of the loaded models
375
- # where the key is the name of the artifact and the value is the path to the model
376
- print(os.listdir(current.model.loaded["my_model"]))
377
- self.next(self.end)
378
- ```
249
+ Specifies that the step will success under all circumstances.
379
250
 
380
- - Loading models
381
- ```python
382
- @step
383
- def train(self):
384
- # current.model.load returns the path to the model loaded
385
- checkpoint_path = current.model.load(
386
- self.checkpoint_key,
387
- )
388
- model_path = current.model.load(
389
- self.model,
390
- )
391
- self.next(self.test)
392
- ```
251
+ The decorator will create an optional artifact, specified by `var`, which
252
+ contains the exception raised. You can use it to detect the presence
253
+ of errors, indicating that all happy-path artifacts produced by the step
254
+ are missing.
393
255
 
394
256
 
395
257
  Parameters
396
258
  ----------
397
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
398
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
399
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
400
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
401
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
402
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
403
-
404
- temp_dir_root : str, default: None
405
- The root directory under which `current.model.loaded` will store loaded models
259
+ var : str, optional, default None
260
+ Name of the artifact in which to store the caught exception.
261
+ If not specified, the exception is not stored.
262
+ print_exception : bool, default True
263
+ Determines whether or not the exception is printed to
264
+ stdout when caught.
406
265
  """
407
266
  ...
408
267
 
409
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
268
+ @typing.overload
269
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
270
+ ...
271
+
272
+ @typing.overload
273
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
274
+ ...
275
+
276
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
410
277
  """
411
- S3 Proxy decorator for routing S3 requests through a local proxy service.
278
+ Specifies that the step will success under all circumstances.
279
+
280
+ The decorator will create an optional artifact, specified by `var`, which
281
+ contains the exception raised. You can use it to detect the presence
282
+ of errors, indicating that all happy-path artifacts produced by the step
283
+ are missing.
412
284
 
413
285
 
414
286
  Parameters
415
287
  ----------
416
- integration_name : str, optional
417
- Name of the S3 proxy integration. If not specified, will use the only
418
- available S3 proxy integration in the namespace (fails if multiple exist).
419
- write_mode : str, optional
420
- The desired behavior during write operations to target (origin) S3 bucket.
421
- allowed options are:
422
- "origin-and-cache" -> write to both the target S3 bucket and local object
423
- storage
424
- "origin" -> only write to the target S3 bucket
425
- "cache" -> only write to the object storage service used for caching
426
- debug : bool, optional
427
- Enable debug logging for proxy operations.
288
+ var : str, optional, default None
289
+ Name of the artifact in which to store the caught exception.
290
+ If not specified, the exception is not stored.
291
+ print_exception : bool, default True
292
+ Determines whether or not the exception is printed to
293
+ stdout when caught.
428
294
  """
429
295
  ...
430
296
 
431
297
  @typing.overload
432
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
298
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
433
299
  """
434
- Specifies secrets to be retrieved and injected as environment variables prior to
435
- the execution of a step.
300
+ Creates a human-readable report, a Metaflow Card, after this step completes.
301
+
302
+ Note that you may add multiple `@card` decorators in a step with different parameters.
436
303
 
437
304
 
438
305
  Parameters
439
306
  ----------
440
- sources : List[Union[str, Dict[str, Any]]], default: []
441
- List of secret specs, defining how the secrets are to be retrieved
442
- role : str, optional, default: None
443
- Role to use for fetching secrets
307
+ type : str, default 'default'
308
+ Card type.
309
+ id : str, optional, default None
310
+ If multiple cards are present, use this id to identify this card.
311
+ options : Dict[str, Any], default {}
312
+ Options passed to the card. The contents depend on the card type.
313
+ timeout : int, default 45
314
+ Interrupt reporting if it takes more than this many seconds.
444
315
  """
445
316
  ...
446
317
 
447
318
  @typing.overload
448
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
319
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
449
320
  ...
450
321
 
451
322
  @typing.overload
452
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
323
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
453
324
  ...
454
325
 
455
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
326
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
456
327
  """
457
- Specifies secrets to be retrieved and injected as environment variables prior to
458
- the execution of a step.
328
+ Creates a human-readable report, a Metaflow Card, after this step completes.
329
+
330
+ Note that you may add multiple `@card` decorators in a step with different parameters.
459
331
 
460
332
 
461
333
  Parameters
462
334
  ----------
463
- sources : List[Union[str, Dict[str, Any]]], default: []
464
- List of secret specs, defining how the secrets are to be retrieved
465
- role : str, optional, default: None
466
- Role to use for fetching secrets
467
- """
468
- ...
469
-
470
- @typing.overload
471
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
472
- """
473
- A simple decorator that demonstrates using CardDecoratorInjector
474
- to inject a card and render simple markdown content.
335
+ type : str, default 'default'
336
+ Card type.
337
+ id : str, optional, default None
338
+ If multiple cards are present, use this id to identify this card.
339
+ options : Dict[str, Any], default {}
340
+ Options passed to the card. The contents depend on the card type.
341
+ timeout : int, default 45
342
+ Interrupt reporting if it takes more than this many seconds.
475
343
  """
476
344
  ...
477
345
 
478
346
  @typing.overload
479
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
480
- ...
481
-
482
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
347
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
483
348
  """
484
- A simple decorator that demonstrates using CardDecoratorInjector
485
- to inject a card and render simple markdown content.
349
+ Specifies the number of times the task corresponding
350
+ to a step needs to be retried.
351
+
352
+ This decorator is useful for handling transient errors, such as networking issues.
353
+ If your task contains operations that can't be retried safely, e.g. database updates,
354
+ it is advisable to annotate it with `@retry(times=0)`.
355
+
356
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
357
+ decorator will execute a no-op task after all retries have been exhausted,
358
+ ensuring that the flow execution can continue.
359
+
360
+
361
+ Parameters
362
+ ----------
363
+ times : int, default 3
364
+ Number of times to retry this task.
365
+ minutes_between_retries : int, default 2
366
+ Number of minutes between retries.
486
367
  """
487
368
  ...
488
369
 
489
370
  @typing.overload
490
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
491
- """
492
- Decorator prototype for all step decorators. This function gets specialized
493
- and imported for all decorators types by _import_plugin_decorators().
494
- """
371
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
495
372
  ...
496
373
 
497
374
  @typing.overload
498
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
375
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
499
376
  ...
500
377
 
501
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
378
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
502
379
  """
503
- Decorator prototype for all step decorators. This function gets specialized
504
- and imported for all decorators types by _import_plugin_decorators().
380
+ Specifies the number of times the task corresponding
381
+ to a step needs to be retried.
382
+
383
+ This decorator is useful for handling transient errors, such as networking issues.
384
+ If your task contains operations that can't be retried safely, e.g. database updates,
385
+ it is advisable to annotate it with `@retry(times=0)`.
386
+
387
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
388
+ decorator will execute a no-op task after all retries have been exhausted,
389
+ ensuring that the flow execution can continue.
390
+
391
+
392
+ Parameters
393
+ ----------
394
+ times : int, default 3
395
+ Number of times to retry this task.
396
+ minutes_between_retries : int, default 2
397
+ Number of minutes between retries.
505
398
  """
506
399
  ...
507
400
 
508
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
401
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
509
402
  """
510
403
  Specifies that this step should execute on DGX cloud.
511
404
 
@@ -516,92 +409,156 @@ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Cal
516
409
  Number of GPUs to use.
517
410
  gpu_type : str
518
411
  Type of Nvidia GPU to use.
412
+ queue_timeout : int
413
+ Time to keep the job in NVCF's queue.
519
414
  """
520
415
  ...
521
416
 
522
417
  @typing.overload
523
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
418
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
524
419
  """
525
- Creates a human-readable report, a Metaflow Card, after this step completes.
526
-
527
- Note that you may add multiple `@card` decorators in a step with different parameters.
420
+ Specifies environment variables to be set prior to the execution of a step.
528
421
 
529
422
 
530
423
  Parameters
531
424
  ----------
532
- type : str, default 'default'
533
- Card type.
534
- id : str, optional, default None
535
- If multiple cards are present, use this id to identify this card.
536
- options : Dict[str, Any], default {}
537
- Options passed to the card. The contents depend on the card type.
538
- timeout : int, default 45
539
- Interrupt reporting if it takes more than this many seconds.
425
+ vars : Dict[str, str], default {}
426
+ Dictionary of environment variables to set.
540
427
  """
541
428
  ...
542
429
 
543
430
  @typing.overload
544
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
431
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
545
432
  ...
546
433
 
547
434
  @typing.overload
548
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
435
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
549
436
  ...
550
437
 
551
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
438
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
552
439
  """
553
- Creates a human-readable report, a Metaflow Card, after this step completes.
440
+ Specifies environment variables to be set prior to the execution of a step.
554
441
 
555
- Note that you may add multiple `@card` decorators in a step with different parameters.
442
+
443
+ Parameters
444
+ ----------
445
+ vars : Dict[str, str], default {}
446
+ Dictionary of environment variables to set.
447
+ """
448
+ ...
449
+
450
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
451
+ """
452
+ Specifies that this step should execute on DGX cloud.
556
453
 
557
454
 
558
455
  Parameters
559
456
  ----------
560
- type : str, default 'default'
561
- Card type.
562
- id : str, optional, default None
563
- If multiple cards are present, use this id to identify this card.
564
- options : Dict[str, Any], default {}
565
- Options passed to the card. The contents depend on the card type.
566
- timeout : int, default 45
567
- Interrupt reporting if it takes more than this many seconds.
457
+ gpu : int
458
+ Number of GPUs to use.
459
+ gpu_type : str
460
+ Type of Nvidia GPU to use.
568
461
  """
569
462
  ...
570
463
 
571
464
  @typing.overload
572
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
465
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
573
466
  """
574
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
575
- It exists to make it easier for users to know that this decorator should only be used with
576
- a Neo Cloud like CoreWeave.
467
+ Specifies the resources needed when executing this step.
468
+
469
+ Use `@resources` to specify the resource requirements
470
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
471
+
472
+ You can choose the compute layer on the command line by executing e.g.
473
+ ```
474
+ python myflow.py run --with batch
475
+ ```
476
+ or
477
+ ```
478
+ python myflow.py run --with kubernetes
479
+ ```
480
+ which executes the flow on the desired system using the
481
+ requirements specified in `@resources`.
482
+
483
+
484
+ Parameters
485
+ ----------
486
+ cpu : int, default 1
487
+ Number of CPUs required for this step.
488
+ gpu : int, optional, default None
489
+ Number of GPUs required for this step.
490
+ disk : int, optional, default None
491
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
492
+ memory : int, default 4096
493
+ Memory size (in MB) required for this step.
494
+ shared_memory : int, optional, default None
495
+ The value for the size (in MiB) of the /dev/shm volume for this step.
496
+ This parameter maps to the `--shm-size` option in Docker.
577
497
  """
578
498
  ...
579
499
 
580
500
  @typing.overload
581
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
501
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
582
502
  ...
583
503
 
584
- def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
504
+ @typing.overload
505
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
506
+ ...
507
+
508
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
585
509
  """
586
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
587
- It exists to make it easier for users to know that this decorator should only be used with
588
- a Neo Cloud like CoreWeave.
510
+ Specifies the resources needed when executing this step.
511
+
512
+ Use `@resources` to specify the resource requirements
513
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
514
+
515
+ You can choose the compute layer on the command line by executing e.g.
516
+ ```
517
+ python myflow.py run --with batch
518
+ ```
519
+ or
520
+ ```
521
+ python myflow.py run --with kubernetes
522
+ ```
523
+ which executes the flow on the desired system using the
524
+ requirements specified in `@resources`.
525
+
526
+
527
+ Parameters
528
+ ----------
529
+ cpu : int, default 1
530
+ Number of CPUs required for this step.
531
+ gpu : int, optional, default None
532
+ Number of GPUs required for this step.
533
+ disk : int, optional, default None
534
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
535
+ memory : int, default 4096
536
+ Memory size (in MB) required for this step.
537
+ shared_memory : int, optional, default None
538
+ The value for the size (in MiB) of the /dev/shm volume for this step.
539
+ This parameter maps to the `--shm-size` option in Docker.
589
540
  """
590
541
  ...
591
542
 
592
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
543
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
593
544
  """
594
- Specifies that this step should execute on DGX cloud.
545
+ S3 Proxy decorator for routing S3 requests through a local proxy service.
595
546
 
596
547
 
597
548
  Parameters
598
549
  ----------
599
- gpu : int
600
- Number of GPUs to use.
601
- gpu_type : str
602
- Type of Nvidia GPU to use.
603
- queue_timeout : int
604
- Time to keep the job in NVCF's queue.
550
+ integration_name : str, optional
551
+ Name of the S3 proxy integration. If not specified, will use the only
552
+ available S3 proxy integration in the namespace (fails if multiple exist).
553
+ write_mode : str, optional
554
+ The desired behavior during write operations to target (origin) S3 bucket.
555
+ allowed options are:
556
+ "origin-and-cache" -> write to both the target S3 bucket and local object
557
+ storage
558
+ "origin" -> only write to the target S3 bucket
559
+ "cache" -> only write to the object storage service used for caching
560
+ debug : bool, optional
561
+ Enable debug logging for proxy operations.
605
562
  """
606
563
  ...
607
564
 
@@ -753,126 +710,61 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
753
710
  ...
754
711
 
755
712
  @typing.overload
756
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
713
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
757
714
  """
758
- Specifies the PyPI packages for the step.
715
+ Specifies a timeout for your step.
759
716
 
760
- Information in this decorator will augment any
761
- attributes set in the `@pyi_base` flow-level decorator. Hence,
762
- you can use `@pypi_base` to set packages required by all
763
- steps and use `@pypi` to specify step-specific overrides.
717
+ This decorator is useful if this step may hang indefinitely.
718
+
719
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
720
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
721
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
722
+
723
+ Note that all the values specified in parameters are added together so if you specify
724
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
764
725
 
765
726
 
766
727
  Parameters
767
728
  ----------
768
- packages : Dict[str, str], default: {}
769
- Packages to use for this step. The key is the name of the package
770
- and the value is the version to use.
771
- python : str, optional, default: None
772
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
773
- that the version used will correspond to the version of the Python interpreter used to start the run.
729
+ seconds : int, default 0
730
+ Number of seconds to wait prior to timing out.
731
+ minutes : int, default 0
732
+ Number of minutes to wait prior to timing out.
733
+ hours : int, default 0
734
+ Number of hours to wait prior to timing out.
774
735
  """
775
736
  ...
776
737
 
777
738
  @typing.overload
778
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
739
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
779
740
  ...
780
741
 
781
742
  @typing.overload
782
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
743
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
783
744
  ...
784
745
 
785
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
746
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
786
747
  """
787
- Specifies the PyPI packages for the step.
748
+ Specifies a timeout for your step.
788
749
 
789
- Information in this decorator will augment any
790
- attributes set in the `@pyi_base` flow-level decorator. Hence,
791
- you can use `@pypi_base` to set packages required by all
792
- steps and use `@pypi` to specify step-specific overrides.
793
-
794
-
795
- Parameters
796
- ----------
797
- packages : Dict[str, str], default: {}
798
- Packages to use for this step. The key is the name of the package
799
- and the value is the version to use.
800
- python : str, optional, default: None
801
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
802
- that the version used will correspond to the version of the Python interpreter used to start the run.
803
- """
804
- ...
805
-
806
- @typing.overload
807
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
808
- """
809
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
810
- It exists to make it easier for users to know that this decorator should only be used with
811
- a Neo Cloud like Nebius.
812
- """
813
- ...
814
-
815
- @typing.overload
816
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
817
- ...
818
-
819
- def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
820
- """
821
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
822
- It exists to make it easier for users to know that this decorator should only be used with
823
- a Neo Cloud like Nebius.
824
- """
825
- ...
826
-
827
- @typing.overload
828
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
829
- """
830
- Decorator prototype for all step decorators. This function gets specialized
831
- and imported for all decorators types by _import_plugin_decorators().
832
- """
833
- ...
834
-
835
- @typing.overload
836
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
837
- ...
838
-
839
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
840
- """
841
- Decorator prototype for all step decorators. This function gets specialized
842
- and imported for all decorators types by _import_plugin_decorators().
843
- """
844
- ...
845
-
846
- @typing.overload
847
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
848
- """
849
- Specifies environment variables to be set prior to the execution of a step.
750
+ This decorator is useful if this step may hang indefinitely.
850
751
 
752
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
753
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
754
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
851
755
 
852
- Parameters
853
- ----------
854
- vars : Dict[str, str], default {}
855
- Dictionary of environment variables to set.
856
- """
857
- ...
858
-
859
- @typing.overload
860
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
861
- ...
862
-
863
- @typing.overload
864
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
865
- ...
866
-
867
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
868
- """
869
- Specifies environment variables to be set prior to the execution of a step.
756
+ Note that all the values specified in parameters are added together so if you specify
757
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
870
758
 
871
759
 
872
760
  Parameters
873
761
  ----------
874
- vars : Dict[str, str], default {}
875
- Dictionary of environment variables to set.
762
+ seconds : int, default 0
763
+ Number of seconds to wait prior to timing out.
764
+ minutes : int, default 0
765
+ Number of minutes to wait prior to timing out.
766
+ hours : int, default 0
767
+ Number of hours to wait prior to timing out.
876
768
  """
877
769
  ...
878
770
 
@@ -920,99 +812,21 @@ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy:
920
812
  ...
921
813
 
922
814
  @typing.overload
923
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
815
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
924
816
  """
925
- Internal decorator to support Fast bakery
817
+ Decorator prototype for all step decorators. This function gets specialized
818
+ and imported for all decorators types by _import_plugin_decorators().
926
819
  """
927
820
  ...
928
821
 
929
822
  @typing.overload
930
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
931
- ...
932
-
933
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
934
- """
935
- Internal decorator to support Fast bakery
936
- """
823
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
937
824
  ...
938
825
 
939
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
826
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
940
827
  """
941
- Decorator that helps cache, version and store models/datasets from huggingface hub.
942
-
943
- > Examples
944
-
945
- **Usage: creating references of models from huggingface that may be loaded in downstream steps**
946
- ```python
947
- @huggingface_hub
948
- @step
949
- def pull_model_from_huggingface(self):
950
- # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
951
- # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
952
- # with the same `repo_id` in the backend storage, it will not download the model again. The return
953
- # value of the function is a reference to the model in the backend storage.
954
- # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
955
-
956
- self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
957
- self.llama_model = current.huggingface_hub.snapshot_download(
958
- repo_id=self.model_id,
959
- allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
960
- )
961
- self.next(self.train)
962
- ```
963
-
964
- **Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
965
- ```python
966
- @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
967
- @step
968
- def pull_model_from_huggingface(self):
969
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
970
- ```
971
-
972
- ```python
973
- @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
974
- @step
975
- def finetune_model(self):
976
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
977
- # path_to_model will be /my-directory
978
- ```
979
-
980
- ```python
981
- # Takes all the arguments passed to `snapshot_download`
982
- # except for `local_dir`
983
- @huggingface_hub(load=[
984
- {
985
- "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
986
- },
987
- {
988
- "repo_id": "myorg/mistral-lora",
989
- "repo_type": "model",
990
- },
991
- ])
992
- @step
993
- def finetune_model(self):
994
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
995
- # path_to_model will be /my-directory
996
- ```
997
-
998
-
999
- Parameters
1000
- ----------
1001
- temp_dir_root : str, optional
1002
- The root directory that will hold the temporary directory where objects will be downloaded.
1003
-
1004
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
1005
- The list of repos (models/datasets) to load.
1006
-
1007
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
1008
-
1009
- - If repo (model/dataset) is not found in the datastore:
1010
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
1011
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
1012
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
1013
-
1014
- - If repo is found in the datastore:
1015
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
828
+ Decorator prototype for all step decorators. This function gets specialized
829
+ and imported for all decorators types by _import_plugin_decorators().
1016
830
  """
1017
831
  ...
1018
832
 
@@ -1105,196 +919,382 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
1105
919
  """
1106
920
  ...
1107
921
 
1108
- @typing.overload
1109
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
922
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1110
923
  """
1111
- Specifies the number of times the task corresponding
1112
- to a step needs to be retried.
1113
-
1114
- This decorator is useful for handling transient errors, such as networking issues.
1115
- If your task contains operations that can't be retried safely, e.g. database updates,
1116
- it is advisable to annotate it with `@retry(times=0)`.
924
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
1117
925
 
1118
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1119
- decorator will execute a no-op task after all retries have been exhausted,
1120
- ensuring that the flow execution can continue.
926
+ > Examples
1121
927
 
928
+ **Usage: creating references of models from huggingface that may be loaded in downstream steps**
929
+ ```python
930
+ @huggingface_hub
931
+ @step
932
+ def pull_model_from_huggingface(self):
933
+ # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
934
+ # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
935
+ # with the same `repo_id` in the backend storage, it will not download the model again. The return
936
+ # value of the function is a reference to the model in the backend storage.
937
+ # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
1122
938
 
1123
- Parameters
1124
- ----------
1125
- times : int, default 3
1126
- Number of times to retry this task.
1127
- minutes_between_retries : int, default 2
1128
- Number of minutes between retries.
1129
- """
1130
- ...
1131
-
1132
- @typing.overload
1133
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1134
- ...
1135
-
1136
- @typing.overload
1137
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1138
- ...
1139
-
1140
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1141
- """
1142
- Specifies the number of times the task corresponding
1143
- to a step needs to be retried.
939
+ self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
940
+ self.llama_model = current.huggingface_hub.snapshot_download(
941
+ repo_id=self.model_id,
942
+ allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
943
+ )
944
+ self.next(self.train)
945
+ ```
1144
946
 
1145
- This decorator is useful for handling transient errors, such as networking issues.
1146
- If your task contains operations that can't be retried safely, e.g. database updates,
1147
- it is advisable to annotate it with `@retry(times=0)`.
947
+ **Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
948
+ ```python
949
+ @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
950
+ @step
951
+ def pull_model_from_huggingface(self):
952
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
953
+ ```
1148
954
 
1149
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1150
- decorator will execute a no-op task after all retries have been exhausted,
1151
- ensuring that the flow execution can continue.
955
+ ```python
956
+ @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
957
+ @step
958
+ def finetune_model(self):
959
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
960
+ # path_to_model will be /my-directory
961
+ ```
962
+
963
+ ```python
964
+ # Takes all the arguments passed to `snapshot_download`
965
+ # except for `local_dir`
966
+ @huggingface_hub(load=[
967
+ {
968
+ "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
969
+ },
970
+ {
971
+ "repo_id": "myorg/mistral-lora",
972
+ "repo_type": "model",
973
+ },
974
+ ])
975
+ @step
976
+ def finetune_model(self):
977
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
978
+ # path_to_model will be /my-directory
979
+ ```
1152
980
 
1153
981
 
1154
982
  Parameters
1155
983
  ----------
1156
- times : int, default 3
1157
- Number of times to retry this task.
1158
- minutes_between_retries : int, default 2
1159
- Number of minutes between retries.
984
+ temp_dir_root : str, optional
985
+ The root directory that will hold the temporary directory where objects will be downloaded.
986
+
987
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
988
+ The list of repos (models/datasets) to load.
989
+
990
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
991
+
992
+ - If repo (model/dataset) is not found in the datastore:
993
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
994
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
995
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
996
+
997
+ - If repo is found in the datastore:
998
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
1160
999
  """
1161
1000
  ...
1162
1001
 
1163
1002
  @typing.overload
1164
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1003
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1165
1004
  """
1166
- Specifies the resources needed when executing this step.
1005
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1006
+ It exists to make it easier for users to know that this decorator should only be used with
1007
+ a Neo Cloud like CoreWeave.
1008
+ """
1009
+ ...
1010
+
1011
+ @typing.overload
1012
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1013
+ ...
1014
+
1015
+ def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1016
+ """
1017
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1018
+ It exists to make it easier for users to know that this decorator should only be used with
1019
+ a Neo Cloud like CoreWeave.
1020
+ """
1021
+ ...
1022
+
1023
+ @typing.overload
1024
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1025
+ """
1026
+ Specifies secrets to be retrieved and injected as environment variables prior to
1027
+ the execution of a step.
1167
1028
 
1168
- Use `@resources` to specify the resource requirements
1169
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1170
1029
 
1171
- You can choose the compute layer on the command line by executing e.g.
1172
- ```
1173
- python myflow.py run --with batch
1174
- ```
1175
- or
1030
+ Parameters
1031
+ ----------
1032
+ sources : List[Union[str, Dict[str, Any]]], default: []
1033
+ List of secret specs, defining how the secrets are to be retrieved
1034
+ role : str, optional, default: None
1035
+ Role to use for fetching secrets
1036
+ """
1037
+ ...
1038
+
1039
+ @typing.overload
1040
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1041
+ ...
1042
+
1043
+ @typing.overload
1044
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1045
+ ...
1046
+
1047
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
1048
+ """
1049
+ Specifies secrets to be retrieved and injected as environment variables prior to
1050
+ the execution of a step.
1051
+
1052
+
1053
+ Parameters
1054
+ ----------
1055
+ sources : List[Union[str, Dict[str, Any]]], default: []
1056
+ List of secret specs, defining how the secrets are to be retrieved
1057
+ role : str, optional, default: None
1058
+ Role to use for fetching secrets
1059
+ """
1060
+ ...
1061
+
1062
+ @typing.overload
1063
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1064
+ """
1065
+ Enables loading / saving of models within a step.
1066
+
1067
+ > Examples
1068
+ - Saving Models
1069
+ ```python
1070
+ @model
1071
+ @step
1072
+ def train(self):
1073
+ # current.model.save returns a dictionary reference to the model saved
1074
+ self.my_model = current.model.save(
1075
+ path_to_my_model,
1076
+ label="my_model",
1077
+ metadata={
1078
+ "epochs": 10,
1079
+ "batch-size": 32,
1080
+ "learning-rate": 0.001,
1081
+ }
1082
+ )
1083
+ self.next(self.test)
1084
+
1085
+ @model(load="my_model")
1086
+ @step
1087
+ def test(self):
1088
+ # `current.model.loaded` returns a dictionary of the loaded models
1089
+ # where the key is the name of the artifact and the value is the path to the model
1090
+ print(os.listdir(current.model.loaded["my_model"]))
1091
+ self.next(self.end)
1176
1092
  ```
1177
- python myflow.py run --with kubernetes
1093
+
1094
+ - Loading models
1095
+ ```python
1096
+ @step
1097
+ def train(self):
1098
+ # current.model.load returns the path to the model loaded
1099
+ checkpoint_path = current.model.load(
1100
+ self.checkpoint_key,
1101
+ )
1102
+ model_path = current.model.load(
1103
+ self.model,
1104
+ )
1105
+ self.next(self.test)
1178
1106
  ```
1179
- which executes the flow on the desired system using the
1180
- requirements specified in `@resources`.
1181
1107
 
1182
1108
 
1183
1109
  Parameters
1184
1110
  ----------
1185
- cpu : int, default 1
1186
- Number of CPUs required for this step.
1187
- gpu : int, optional, default None
1188
- Number of GPUs required for this step.
1189
- disk : int, optional, default None
1190
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1191
- memory : int, default 4096
1192
- Memory size (in MB) required for this step.
1193
- shared_memory : int, optional, default None
1194
- The value for the size (in MiB) of the /dev/shm volume for this step.
1195
- This parameter maps to the `--shm-size` option in Docker.
1111
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1112
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1113
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1114
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1115
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1116
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1117
+
1118
+ temp_dir_root : str, default: None
1119
+ The root directory under which `current.model.loaded` will store loaded models
1196
1120
  """
1197
1121
  ...
1198
1122
 
1199
1123
  @typing.overload
1200
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1124
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1201
1125
  ...
1202
1126
 
1203
1127
  @typing.overload
1204
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1128
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1205
1129
  ...
1206
1130
 
1207
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1131
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1208
1132
  """
1209
- Specifies the resources needed when executing this step.
1133
+ Enables loading / saving of models within a step.
1210
1134
 
1211
- Use `@resources` to specify the resource requirements
1212
- independently of the specific compute layer (`@batch`, `@kubernetes`).
1135
+ > Examples
1136
+ - Saving Models
1137
+ ```python
1138
+ @model
1139
+ @step
1140
+ def train(self):
1141
+ # current.model.save returns a dictionary reference to the model saved
1142
+ self.my_model = current.model.save(
1143
+ path_to_my_model,
1144
+ label="my_model",
1145
+ metadata={
1146
+ "epochs": 10,
1147
+ "batch-size": 32,
1148
+ "learning-rate": 0.001,
1149
+ }
1150
+ )
1151
+ self.next(self.test)
1213
1152
 
1214
- You can choose the compute layer on the command line by executing e.g.
1215
- ```
1216
- python myflow.py run --with batch
1217
- ```
1218
- or
1153
+ @model(load="my_model")
1154
+ @step
1155
+ def test(self):
1156
+ # `current.model.loaded` returns a dictionary of the loaded models
1157
+ # where the key is the name of the artifact and the value is the path to the model
1158
+ print(os.listdir(current.model.loaded["my_model"]))
1159
+ self.next(self.end)
1219
1160
  ```
1220
- python myflow.py run --with kubernetes
1161
+
1162
+ - Loading models
1163
+ ```python
1164
+ @step
1165
+ def train(self):
1166
+ # current.model.load returns the path to the model loaded
1167
+ checkpoint_path = current.model.load(
1168
+ self.checkpoint_key,
1169
+ )
1170
+ model_path = current.model.load(
1171
+ self.model,
1172
+ )
1173
+ self.next(self.test)
1221
1174
  ```
1222
- which executes the flow on the desired system using the
1223
- requirements specified in `@resources`.
1224
1175
 
1225
1176
 
1226
1177
  Parameters
1227
1178
  ----------
1228
- cpu : int, default 1
1229
- Number of CPUs required for this step.
1230
- gpu : int, optional, default None
1231
- Number of GPUs required for this step.
1232
- disk : int, optional, default None
1233
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1234
- memory : int, default 4096
1235
- Memory size (in MB) required for this step.
1236
- shared_memory : int, optional, default None
1237
- The value for the size (in MiB) of the /dev/shm volume for this step.
1238
- This parameter maps to the `--shm-size` option in Docker.
1179
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1180
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1181
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1182
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1183
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1184
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1185
+
1186
+ temp_dir_root : str, default: None
1187
+ The root directory under which `current.model.loaded` will store loaded models
1239
1188
  """
1240
1189
  ...
1241
1190
 
1242
1191
  @typing.overload
1243
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1192
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1244
1193
  """
1245
- Specifies a timeout for your step.
1194
+ Decorator prototype for all step decorators. This function gets specialized
1195
+ and imported for all decorators types by _import_plugin_decorators().
1196
+ """
1197
+ ...
1198
+
1199
+ @typing.overload
1200
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1201
+ ...
1202
+
1203
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1204
+ """
1205
+ Decorator prototype for all step decorators. This function gets specialized
1206
+ and imported for all decorators types by _import_plugin_decorators().
1207
+ """
1208
+ ...
1209
+
1210
+ @typing.overload
1211
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1212
+ """
1213
+ Specifies the PyPI packages for the step.
1246
1214
 
1247
- This decorator is useful if this step may hang indefinitely.
1215
+ Information in this decorator will augment any
1216
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1217
+ you can use `@pypi_base` to set packages required by all
1218
+ steps and use `@pypi` to specify step-specific overrides.
1248
1219
 
1249
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1250
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1251
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1252
1220
 
1253
- Note that all the values specified in parameters are added together so if you specify
1254
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1221
+ Parameters
1222
+ ----------
1223
+ packages : Dict[str, str], default: {}
1224
+ Packages to use for this step. The key is the name of the package
1225
+ and the value is the version to use.
1226
+ python : str, optional, default: None
1227
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1228
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1229
+ """
1230
+ ...
1231
+
1232
+ @typing.overload
1233
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1234
+ ...
1235
+
1236
+ @typing.overload
1237
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1238
+ ...
1239
+
1240
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1241
+ """
1242
+ Specifies the PyPI packages for the step.
1243
+
1244
+ Information in this decorator will augment any
1245
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1246
+ you can use `@pypi_base` to set packages required by all
1247
+ steps and use `@pypi` to specify step-specific overrides.
1255
1248
 
1256
1249
 
1257
1250
  Parameters
1258
1251
  ----------
1259
- seconds : int, default 0
1260
- Number of seconds to wait prior to timing out.
1261
- minutes : int, default 0
1262
- Number of minutes to wait prior to timing out.
1263
- hours : int, default 0
1264
- Number of hours to wait prior to timing out.
1252
+ packages : Dict[str, str], default: {}
1253
+ Packages to use for this step. The key is the name of the package
1254
+ and the value is the version to use.
1255
+ python : str, optional, default: None
1256
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1257
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1258
+ """
1259
+ ...
1260
+
1261
+ @typing.overload
1262
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1263
+ """
1264
+ A simple decorator that demonstrates using CardDecoratorInjector
1265
+ to inject a card and render simple markdown content.
1266
+ """
1267
+ ...
1268
+
1269
+ @typing.overload
1270
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1271
+ ...
1272
+
1273
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1274
+ """
1275
+ A simple decorator that demonstrates using CardDecoratorInjector
1276
+ to inject a card and render simple markdown content.
1277
+ """
1278
+ ...
1279
+
1280
+ @typing.overload
1281
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1282
+ """
1283
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1284
+ It exists to make it easier for users to know that this decorator should only be used with
1285
+ a Neo Cloud like Nebius.
1265
1286
  """
1266
1287
  ...
1267
1288
 
1268
1289
  @typing.overload
1269
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1270
- ...
1271
-
1272
- @typing.overload
1273
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1290
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1274
1291
  ...
1275
1292
 
1276
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1293
+ def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1277
1294
  """
1278
- Specifies a timeout for your step.
1279
-
1280
- This decorator is useful if this step may hang indefinitely.
1281
-
1282
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1283
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1284
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1285
-
1286
- Note that all the values specified in parameters are added together so if you specify
1287
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1288
-
1289
-
1290
- Parameters
1291
- ----------
1292
- seconds : int, default 0
1293
- Number of seconds to wait prior to timing out.
1294
- minutes : int, default 0
1295
- Number of minutes to wait prior to timing out.
1296
- hours : int, default 0
1297
- Number of hours to wait prior to timing out.
1295
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1296
+ It exists to make it easier for users to know that this decorator should only be used with
1297
+ a Neo Cloud like Nebius.
1298
1298
  """
1299
1299
  ...
1300
1300
 
@@ -1348,233 +1348,182 @@ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card
1348
1348
  """
1349
1349
  ...
1350
1350
 
1351
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1352
- """
1353
- Specifies what flows belong to the same project.
1354
-
1355
- A project-specific namespace is created for all flows that
1356
- use the same `@project(name)`.
1357
-
1358
-
1359
- Parameters
1360
- ----------
1361
- name : str
1362
- Project name. Make sure that the name is unique amongst all
1363
- projects that use the same production scheduler. The name may
1364
- contain only lowercase alphanumeric characters and underscores.
1365
-
1366
- branch : Optional[str], default None
1367
- The branch to use. If not specified, the branch is set to
1368
- `user.<username>` unless `production` is set to `True`. This can
1369
- also be set on the command line using `--branch` as a top-level option.
1370
- It is an error to specify `branch` in the decorator and on the command line.
1371
-
1372
- production : bool, default False
1373
- Whether or not the branch is the production branch. This can also be set on the
1374
- command line using `--production` as a top-level option. It is an error to specify
1375
- `production` in the decorator and on the command line.
1376
- The project branch name will be:
1377
- - if `branch` is specified:
1378
- - if `production` is True: `prod.<branch>`
1379
- - if `production` is False: `test.<branch>`
1380
- - if `branch` is not specified:
1381
- - if `production` is True: `prod`
1382
- - if `production` is False: `user.<username>`
1383
- """
1384
- ...
1385
-
1386
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1387
- """
1388
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1389
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1390
-
1391
-
1392
- Parameters
1393
- ----------
1394
- timeout : int
1395
- Time, in seconds before the task times out and fails. (Default: 3600)
1396
- poke_interval : int
1397
- Time in seconds that the job should wait in between each try. (Default: 60)
1398
- mode : str
1399
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1400
- exponential_backoff : bool
1401
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1402
- pool : str
1403
- the slot pool this task should run in,
1404
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1405
- soft_fail : bool
1406
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1407
- name : str
1408
- Name of the sensor on Airflow
1409
- description : str
1410
- Description of sensor in the Airflow UI
1411
- external_dag_id : str
1412
- The dag_id that contains the task you want to wait for.
1413
- external_task_ids : List[str]
1414
- The list of task_ids that you want to wait for.
1415
- If None (default value) the sensor waits for the DAG. (Default: None)
1416
- allowed_states : List[str]
1417
- Iterable of allowed states, (Default: ['success'])
1418
- failed_states : List[str]
1419
- Iterable of failed or dis-allowed states. (Default: None)
1420
- execution_delta : datetime.timedelta
1421
- time difference with the previous execution to look at,
1422
- the default is the same logical date as the current task or DAG. (Default: None)
1423
- check_existence: bool
1424
- Set to True to check if the external task exists or check if
1425
- the DAG to wait for exists. (Default: True)
1426
- """
1427
- ...
1428
-
1429
1351
  @typing.overload
1430
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1352
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1431
1353
  """
1432
- Specifies the times when the flow should be run when running on a
1433
- production scheduler.
1354
+ Specifies the Conda environment for all steps of the flow.
1355
+
1356
+ Use `@conda_base` to set common libraries required by all
1357
+ steps and use `@conda` to specify step-specific additions.
1434
1358
 
1435
1359
 
1436
1360
  Parameters
1437
1361
  ----------
1438
- hourly : bool, default False
1439
- Run the workflow hourly.
1440
- daily : bool, default True
1441
- Run the workflow daily.
1442
- weekly : bool, default False
1443
- Run the workflow weekly.
1444
- cron : str, optional, default None
1445
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1446
- specified by this expression.
1447
- timezone : str, optional, default None
1448
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1449
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1362
+ packages : Dict[str, str], default {}
1363
+ Packages to use for this flow. The key is the name of the package
1364
+ and the value is the version to use.
1365
+ libraries : Dict[str, str], default {}
1366
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1367
+ python : str, optional, default None
1368
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1369
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1370
+ disabled : bool, default False
1371
+ If set to True, disables Conda.
1450
1372
  """
1451
1373
  ...
1452
1374
 
1453
1375
  @typing.overload
1454
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1376
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1455
1377
  ...
1456
1378
 
1457
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1379
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1458
1380
  """
1459
- Specifies the times when the flow should be run when running on a
1460
- production scheduler.
1381
+ Specifies the Conda environment for all steps of the flow.
1382
+
1383
+ Use `@conda_base` to set common libraries required by all
1384
+ steps and use `@conda` to specify step-specific additions.
1461
1385
 
1462
1386
 
1463
1387
  Parameters
1464
1388
  ----------
1465
- hourly : bool, default False
1466
- Run the workflow hourly.
1467
- daily : bool, default True
1468
- Run the workflow daily.
1469
- weekly : bool, default False
1470
- Run the workflow weekly.
1471
- cron : str, optional, default None
1472
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1473
- specified by this expression.
1474
- timezone : str, optional, default None
1475
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1476
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1389
+ packages : Dict[str, str], default {}
1390
+ Packages to use for this flow. The key is the name of the package
1391
+ and the value is the version to use.
1392
+ libraries : Dict[str, str], default {}
1393
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1394
+ python : str, optional, default None
1395
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1396
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1397
+ disabled : bool, default False
1398
+ If set to True, disables Conda.
1477
1399
  """
1478
1400
  ...
1479
1401
 
1480
1402
  @typing.overload
1481
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1403
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1482
1404
  """
1483
- Specifies the flow(s) that this flow depends on.
1405
+ Specifies the event(s) that this flow depends on.
1484
1406
 
1485
1407
  ```
1486
- @trigger_on_finish(flow='FooFlow')
1408
+ @trigger(event='foo')
1487
1409
  ```
1488
1410
  or
1489
1411
  ```
1490
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1412
+ @trigger(events=['foo', 'bar'])
1491
1413
  ```
1492
- This decorator respects the @project decorator and triggers the flow
1493
- when upstream runs within the same namespace complete successfully
1494
1414
 
1495
- Additionally, you can specify project aware upstream flow dependencies
1496
- by specifying the fully qualified project_flow_name.
1415
+ Additionally, you can specify the parameter mappings
1416
+ to map event payload to Metaflow parameters for the flow.
1497
1417
  ```
1498
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1418
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1499
1419
  ```
1500
1420
  or
1501
1421
  ```
1502
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1422
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1423
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1503
1424
  ```
1504
1425
 
1505
- You can also specify just the project or project branch (other values will be
1506
- inferred from the current project or project branch):
1426
+ 'parameters' can also be a list of strings and tuples like so:
1507
1427
  ```
1508
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1428
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1429
+ ```
1430
+ This is equivalent to:
1431
+ ```
1432
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1509
1433
  ```
1510
-
1511
- Note that `branch` is typically one of:
1512
- - `prod`
1513
- - `user.bob`
1514
- - `test.my_experiment`
1515
- - `prod.staging`
1516
1434
 
1517
1435
 
1518
1436
  Parameters
1519
1437
  ----------
1520
- flow : Union[str, Dict[str, str]], optional, default None
1521
- Upstream flow dependency for this flow.
1522
- flows : List[Union[str, Dict[str, str]]], default []
1523
- Upstream flow dependencies for this flow.
1438
+ event : Union[str, Dict[str, Any]], optional, default None
1439
+ Event dependency for this flow.
1440
+ events : List[Union[str, Dict[str, Any]]], default []
1441
+ Events dependency for this flow.
1524
1442
  options : Dict[str, Any], default {}
1525
1443
  Backend-specific configuration for tuning eventing behavior.
1526
1444
  """
1527
1445
  ...
1528
1446
 
1529
1447
  @typing.overload
1530
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1448
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1531
1449
  ...
1532
1450
 
1533
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1451
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1534
1452
  """
1535
- Specifies the flow(s) that this flow depends on.
1453
+ Specifies the event(s) that this flow depends on.
1536
1454
 
1537
1455
  ```
1538
- @trigger_on_finish(flow='FooFlow')
1456
+ @trigger(event='foo')
1539
1457
  ```
1540
1458
  or
1541
1459
  ```
1542
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1460
+ @trigger(events=['foo', 'bar'])
1543
1461
  ```
1544
- This decorator respects the @project decorator and triggers the flow
1545
- when upstream runs within the same namespace complete successfully
1546
1462
 
1547
- Additionally, you can specify project aware upstream flow dependencies
1548
- by specifying the fully qualified project_flow_name.
1463
+ Additionally, you can specify the parameter mappings
1464
+ to map event payload to Metaflow parameters for the flow.
1549
1465
  ```
1550
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1466
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1551
1467
  ```
1552
1468
  or
1553
1469
  ```
1554
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1470
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1471
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1555
1472
  ```
1556
1473
 
1557
- You can also specify just the project or project branch (other values will be
1558
- inferred from the current project or project branch):
1474
+ 'parameters' can also be a list of strings and tuples like so:
1559
1475
  ```
1560
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1476
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1561
1477
  ```
1478
+ This is equivalent to:
1479
+ ```
1480
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1481
+ ```
1482
+
1483
+
1484
+ Parameters
1485
+ ----------
1486
+ event : Union[str, Dict[str, Any]], optional, default None
1487
+ Event dependency for this flow.
1488
+ events : List[Union[str, Dict[str, Any]]], default []
1489
+ Events dependency for this flow.
1490
+ options : Dict[str, Any], default {}
1491
+ Backend-specific configuration for tuning eventing behavior.
1492
+ """
1493
+ ...
1494
+
1495
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1496
+ """
1497
+ Specifies what flows belong to the same project.
1562
1498
 
1563
- Note that `branch` is typically one of:
1564
- - `prod`
1565
- - `user.bob`
1566
- - `test.my_experiment`
1567
- - `prod.staging`
1499
+ A project-specific namespace is created for all flows that
1500
+ use the same `@project(name)`.
1568
1501
 
1569
1502
 
1570
1503
  Parameters
1571
1504
  ----------
1572
- flow : Union[str, Dict[str, str]], optional, default None
1573
- Upstream flow dependency for this flow.
1574
- flows : List[Union[str, Dict[str, str]]], default []
1575
- Upstream flow dependencies for this flow.
1576
- options : Dict[str, Any], default {}
1577
- Backend-specific configuration for tuning eventing behavior.
1505
+ name : str
1506
+ Project name. Make sure that the name is unique amongst all
1507
+ projects that use the same production scheduler. The name may
1508
+ contain only lowercase alphanumeric characters and underscores.
1509
+
1510
+ branch : Optional[str], default None
1511
+ The branch to use. If not specified, the branch is set to
1512
+ `user.<username>` unless `production` is set to `True`. This can
1513
+ also be set on the command line using `--branch` as a top-level option.
1514
+ It is an error to specify `branch` in the decorator and on the command line.
1515
+
1516
+ production : bool, default False
1517
+ Whether or not the branch is the production branch. This can also be set on the
1518
+ command line using `--production` as a top-level option. It is an error to specify
1519
+ `production` in the decorator and on the command line.
1520
+ The project branch name will be:
1521
+ - if `branch` is specified:
1522
+ - if `production` is True: `prod.<branch>`
1523
+ - if `production` is False: `test.<branch>`
1524
+ - if `branch` is not specified:
1525
+ - if `production` is True: `prod`
1526
+ - if `production` is False: `user.<username>`
1578
1527
  """
1579
1528
  ...
1580
1529
 
@@ -1692,6 +1641,49 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1692
1641
  """
1693
1642
  ...
1694
1643
 
1644
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1645
+ """
1646
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1647
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1648
+
1649
+
1650
+ Parameters
1651
+ ----------
1652
+ timeout : int
1653
+ Time, in seconds before the task times out and fails. (Default: 3600)
1654
+ poke_interval : int
1655
+ Time in seconds that the job should wait in between each try. (Default: 60)
1656
+ mode : str
1657
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1658
+ exponential_backoff : bool
1659
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1660
+ pool : str
1661
+ the slot pool this task should run in,
1662
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1663
+ soft_fail : bool
1664
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1665
+ name : str
1666
+ Name of the sensor on Airflow
1667
+ description : str
1668
+ Description of sensor in the Airflow UI
1669
+ external_dag_id : str
1670
+ The dag_id that contains the task you want to wait for.
1671
+ external_task_ids : List[str]
1672
+ The list of task_ids that you want to wait for.
1673
+ If None (default value) the sensor waits for the DAG. (Default: None)
1674
+ allowed_states : List[str]
1675
+ Iterable of allowed states, (Default: ['success'])
1676
+ failed_states : List[str]
1677
+ Iterable of failed or dis-allowed states. (Default: None)
1678
+ execution_delta : datetime.timedelta
1679
+ time difference with the previous execution to look at,
1680
+ the default is the same logical date as the current task or DAG. (Default: None)
1681
+ check_existence: bool
1682
+ Set to True to check if the external task exists or check if
1683
+ the DAG to wait for exists. (Default: True)
1684
+ """
1685
+ ...
1686
+
1695
1687
  def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1696
1688
  """
1697
1689
  The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
@@ -1736,187 +1728,195 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1736
1728
  ...
1737
1729
 
1738
1730
  @typing.overload
1739
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1731
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1740
1732
  """
1741
- Specifies the event(s) that this flow depends on.
1733
+ Specifies the flow(s) that this flow depends on.
1742
1734
 
1743
1735
  ```
1744
- @trigger(event='foo')
1736
+ @trigger_on_finish(flow='FooFlow')
1745
1737
  ```
1746
1738
  or
1747
1739
  ```
1748
- @trigger(events=['foo', 'bar'])
1740
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1749
1741
  ```
1742
+ This decorator respects the @project decorator and triggers the flow
1743
+ when upstream runs within the same namespace complete successfully
1750
1744
 
1751
- Additionally, you can specify the parameter mappings
1752
- to map event payload to Metaflow parameters for the flow.
1745
+ Additionally, you can specify project aware upstream flow dependencies
1746
+ by specifying the fully qualified project_flow_name.
1753
1747
  ```
1754
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1748
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1755
1749
  ```
1756
1750
  or
1757
1751
  ```
1758
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1759
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1752
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1760
1753
  ```
1761
1754
 
1762
- 'parameters' can also be a list of strings and tuples like so:
1763
- ```
1764
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1765
- ```
1766
- This is equivalent to:
1755
+ You can also specify just the project or project branch (other values will be
1756
+ inferred from the current project or project branch):
1767
1757
  ```
1768
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1758
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1769
1759
  ```
1770
1760
 
1761
+ Note that `branch` is typically one of:
1762
+ - `prod`
1763
+ - `user.bob`
1764
+ - `test.my_experiment`
1765
+ - `prod.staging`
1766
+
1771
1767
 
1772
1768
  Parameters
1773
1769
  ----------
1774
- event : Union[str, Dict[str, Any]], optional, default None
1775
- Event dependency for this flow.
1776
- events : List[Union[str, Dict[str, Any]]], default []
1777
- Events dependency for this flow.
1770
+ flow : Union[str, Dict[str, str]], optional, default None
1771
+ Upstream flow dependency for this flow.
1772
+ flows : List[Union[str, Dict[str, str]]], default []
1773
+ Upstream flow dependencies for this flow.
1778
1774
  options : Dict[str, Any], default {}
1779
1775
  Backend-specific configuration for tuning eventing behavior.
1780
1776
  """
1781
1777
  ...
1782
1778
 
1783
1779
  @typing.overload
1784
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1780
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1785
1781
  ...
1786
1782
 
1787
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1783
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1788
1784
  """
1789
- Specifies the event(s) that this flow depends on.
1785
+ Specifies the flow(s) that this flow depends on.
1790
1786
 
1791
1787
  ```
1792
- @trigger(event='foo')
1788
+ @trigger_on_finish(flow='FooFlow')
1793
1789
  ```
1794
1790
  or
1795
1791
  ```
1796
- @trigger(events=['foo', 'bar'])
1792
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1797
1793
  ```
1794
+ This decorator respects the @project decorator and triggers the flow
1795
+ when upstream runs within the same namespace complete successfully
1798
1796
 
1799
- Additionally, you can specify the parameter mappings
1800
- to map event payload to Metaflow parameters for the flow.
1797
+ Additionally, you can specify project aware upstream flow dependencies
1798
+ by specifying the fully qualified project_flow_name.
1801
1799
  ```
1802
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1800
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1803
1801
  ```
1804
1802
  or
1805
1803
  ```
1806
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1807
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1804
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1808
1805
  ```
1809
1806
 
1810
- 'parameters' can also be a list of strings and tuples like so:
1811
- ```
1812
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1813
- ```
1814
- This is equivalent to:
1807
+ You can also specify just the project or project branch (other values will be
1808
+ inferred from the current project or project branch):
1815
1809
  ```
1816
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1810
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1817
1811
  ```
1818
1812
 
1813
+ Note that `branch` is typically one of:
1814
+ - `prod`
1815
+ - `user.bob`
1816
+ - `test.my_experiment`
1817
+ - `prod.staging`
1818
+
1819
1819
 
1820
1820
  Parameters
1821
1821
  ----------
1822
- event : Union[str, Dict[str, Any]], optional, default None
1823
- Event dependency for this flow.
1824
- events : List[Union[str, Dict[str, Any]]], default []
1825
- Events dependency for this flow.
1822
+ flow : Union[str, Dict[str, str]], optional, default None
1823
+ Upstream flow dependency for this flow.
1824
+ flows : List[Union[str, Dict[str, str]]], default []
1825
+ Upstream flow dependencies for this flow.
1826
1826
  options : Dict[str, Any], default {}
1827
1827
  Backend-specific configuration for tuning eventing behavior.
1828
1828
  """
1829
1829
  ...
1830
1830
 
1831
1831
  @typing.overload
1832
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1832
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1833
1833
  """
1834
- Specifies the Conda environment for all steps of the flow.
1835
-
1836
- Use `@conda_base` to set common libraries required by all
1837
- steps and use `@conda` to specify step-specific additions.
1834
+ Specifies the PyPI packages for all steps of the flow.
1838
1835
 
1836
+ Use `@pypi_base` to set common packages required by all
1837
+ steps and use `@pypi` to specify step-specific overrides.
1839
1838
 
1840
1839
  Parameters
1841
1840
  ----------
1842
- packages : Dict[str, str], default {}
1841
+ packages : Dict[str, str], default: {}
1843
1842
  Packages to use for this flow. The key is the name of the package
1844
1843
  and the value is the version to use.
1845
- libraries : Dict[str, str], default {}
1846
- Supported for backward compatibility. When used with packages, packages will take precedence.
1847
- python : str, optional, default None
1844
+ python : str, optional, default: None
1848
1845
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1849
1846
  that the version used will correspond to the version of the Python interpreter used to start the run.
1850
- disabled : bool, default False
1851
- If set to True, disables Conda.
1852
1847
  """
1853
1848
  ...
1854
1849
 
1855
1850
  @typing.overload
1856
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1851
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1857
1852
  ...
1858
1853
 
1859
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1854
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1860
1855
  """
1861
- Specifies the Conda environment for all steps of the flow.
1862
-
1863
- Use `@conda_base` to set common libraries required by all
1864
- steps and use `@conda` to specify step-specific additions.
1856
+ Specifies the PyPI packages for all steps of the flow.
1865
1857
 
1858
+ Use `@pypi_base` to set common packages required by all
1859
+ steps and use `@pypi` to specify step-specific overrides.
1866
1860
 
1867
1861
  Parameters
1868
1862
  ----------
1869
- packages : Dict[str, str], default {}
1863
+ packages : Dict[str, str], default: {}
1870
1864
  Packages to use for this flow. The key is the name of the package
1871
1865
  and the value is the version to use.
1872
- libraries : Dict[str, str], default {}
1873
- Supported for backward compatibility. When used with packages, packages will take precedence.
1874
- python : str, optional, default None
1866
+ python : str, optional, default: None
1875
1867
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1876
1868
  that the version used will correspond to the version of the Python interpreter used to start the run.
1877
- disabled : bool, default False
1878
- If set to True, disables Conda.
1879
1869
  """
1880
1870
  ...
1881
1871
 
1882
1872
  @typing.overload
1883
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1873
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1884
1874
  """
1885
- Specifies the PyPI packages for all steps of the flow.
1875
+ Specifies the times when the flow should be run when running on a
1876
+ production scheduler.
1886
1877
 
1887
- Use `@pypi_base` to set common packages required by all
1888
- steps and use `@pypi` to specify step-specific overrides.
1889
1878
 
1890
1879
  Parameters
1891
1880
  ----------
1892
- packages : Dict[str, str], default: {}
1893
- Packages to use for this flow. The key is the name of the package
1894
- and the value is the version to use.
1895
- python : str, optional, default: None
1896
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1897
- that the version used will correspond to the version of the Python interpreter used to start the run.
1881
+ hourly : bool, default False
1882
+ Run the workflow hourly.
1883
+ daily : bool, default True
1884
+ Run the workflow daily.
1885
+ weekly : bool, default False
1886
+ Run the workflow weekly.
1887
+ cron : str, optional, default None
1888
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1889
+ specified by this expression.
1890
+ timezone : str, optional, default None
1891
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1892
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1898
1893
  """
1899
1894
  ...
1900
1895
 
1901
1896
  @typing.overload
1902
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1897
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1903
1898
  ...
1904
1899
 
1905
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1900
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1906
1901
  """
1907
- Specifies the PyPI packages for all steps of the flow.
1902
+ Specifies the times when the flow should be run when running on a
1903
+ production scheduler.
1908
1904
 
1909
- Use `@pypi_base` to set common packages required by all
1910
- steps and use `@pypi` to specify step-specific overrides.
1911
1905
 
1912
1906
  Parameters
1913
1907
  ----------
1914
- packages : Dict[str, str], default: {}
1915
- Packages to use for this flow. The key is the name of the package
1916
- and the value is the version to use.
1917
- python : str, optional, default: None
1918
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1919
- that the version used will correspond to the version of the Python interpreter used to start the run.
1908
+ hourly : bool, default False
1909
+ Run the workflow hourly.
1910
+ daily : bool, default True
1911
+ Run the workflow daily.
1912
+ weekly : bool, default False
1913
+ Run the workflow weekly.
1914
+ cron : str, optional, default None
1915
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1916
+ specified by this expression.
1917
+ timezone : str, optional, default None
1918
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1919
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1920
1920
  """
1921
1921
  ...
1922
1922