ob-metaflow-stubs 6.0.10.8__py2.py3-none-any.whl → 6.0.10.10__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow-stubs might be problematic. Click here for more details.

Files changed (262) hide show
  1. metaflow-stubs/__init__.pyi +1035 -921
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +4 -4
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +56 -59
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +110 -97
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +4 -4
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +3 -3
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +3 -3
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +3 -3
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +3 -3
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +5 -5
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +4 -4
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +3 -3
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +4 -4
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +4 -4
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +3 -3
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  110. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  111. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  112. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +3 -3
  113. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  114. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +58 -2
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
  116. metaflow-stubs/multicore_utils.pyi +2 -2
  117. metaflow-stubs/ob_internal.pyi +2 -2
  118. metaflow-stubs/packaging_sys/__init__.pyi +7 -7
  119. metaflow-stubs/packaging_sys/backend.pyi +4 -4
  120. metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
  121. metaflow-stubs/packaging_sys/tar_backend.pyi +5 -5
  122. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  123. metaflow-stubs/packaging_sys/v1.pyi +3 -3
  124. metaflow-stubs/parameters.pyi +4 -4
  125. metaflow-stubs/plugins/__init__.pyi +11 -11
  126. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  128. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  129. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  130. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  131. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  132. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  133. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  134. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  135. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  136. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  137. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  138. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  139. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  140. metaflow-stubs/plugins/argo/exit_hooks.pyi +2 -2
  141. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  142. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  143. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  144. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  146. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  147. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  148. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  150. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  152. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  154. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  155. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  157. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  158. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  159. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  160. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  161. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  162. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  163. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  164. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  165. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  166. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  167. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  168. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  169. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  170. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  171. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  172. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  173. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  174. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  175. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  176. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  177. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  178. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  179. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  180. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  181. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  182. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  183. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  184. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  185. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  186. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  187. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  188. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  189. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  190. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  191. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  192. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  193. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  194. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  195. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  196. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  197. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  198. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  199. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  200. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  201. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  202. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  203. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  204. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  205. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  206. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  207. metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
  208. metaflow-stubs/plugins/optuna/__init__.pyi +2 -2
  209. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  210. metaflow-stubs/plugins/perimeters.pyi +2 -2
  211. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  212. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  213. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  214. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  215. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  216. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  217. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  218. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  219. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  220. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  221. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  222. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  223. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  224. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  225. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  226. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  227. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  228. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  229. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  230. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  231. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  232. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  233. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  234. metaflow-stubs/profilers/__init__.pyi +2 -2
  235. metaflow-stubs/pylint_wrapper.pyi +2 -2
  236. metaflow-stubs/runner/__init__.pyi +2 -2
  237. metaflow-stubs/runner/deployer.pyi +33 -33
  238. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  239. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  240. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  241. metaflow-stubs/runner/nbrun.pyi +2 -2
  242. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  243. metaflow-stubs/runner/utils.pyi +3 -3
  244. metaflow-stubs/system/__init__.pyi +2 -2
  245. metaflow-stubs/system/system_logger.pyi +3 -3
  246. metaflow-stubs/system/system_monitor.pyi +2 -2
  247. metaflow-stubs/tagging_util.pyi +2 -2
  248. metaflow-stubs/tuple_util.pyi +2 -2
  249. metaflow-stubs/user_configs/__init__.pyi +2 -2
  250. metaflow-stubs/user_configs/config_options.pyi +3 -3
  251. metaflow-stubs/user_configs/config_parameters.pyi +7 -7
  252. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  253. metaflow-stubs/user_decorators/common.pyi +2 -2
  254. metaflow-stubs/user_decorators/mutable_flow.pyi +6 -6
  255. metaflow-stubs/user_decorators/mutable_step.pyi +4 -4
  256. metaflow-stubs/user_decorators/user_flow_decorator.pyi +4 -4
  257. metaflow-stubs/user_decorators/user_step_decorator.pyi +5 -5
  258. {ob_metaflow_stubs-6.0.10.8.dist-info → ob_metaflow_stubs-6.0.10.10.dist-info}/METADATA +1 -1
  259. ob_metaflow_stubs-6.0.10.10.dist-info/RECORD +262 -0
  260. ob_metaflow_stubs-6.0.10.8.dist-info/RECORD +0 -262
  261. {ob_metaflow_stubs-6.0.10.8.dist-info → ob_metaflow_stubs-6.0.10.10.dist-info}/WHEEL +0 -0
  262. {ob_metaflow_stubs-6.0.10.8.dist-info → ob_metaflow_stubs-6.0.10.10.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.18.7.5+obcheckpoint(0.2.6);ob(v1) #
4
- # Generated on 2025-09-19T21:56:58.875223 #
3
+ # MF version: 2.18.7.5+obcheckpoint(0.2.7);ob(v1) #
4
+ # Generated on 2025-09-23T01:34:30.897811 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -39,18 +39,18 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
39
39
  from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
- from . import metaflow_git as metaflow_git
43
42
  from . import tuple_util as tuple_util
44
43
  from . import cards as cards
44
+ from . import metaflow_git as metaflow_git
45
45
  from . import events as events
46
46
  from . import runner as runner
47
47
  from . import plugins as plugins
48
48
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
49
49
  from . import includefile as includefile
50
50
  from .includefile import IncludeFile as IncludeFile
51
- from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
52
51
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
53
52
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
53
+ from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
54
54
  from . import client as client
55
55
  from .client.core import namespace as namespace
56
56
  from .client.core import get_namespace as get_namespace
@@ -77,6 +77,7 @@ from .mf_extensions.outerbounds.toplevel.s3_proxy import get_S3_with_s3_proxy as
77
77
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import set_s3_proxy_config as set_s3_proxy_config
78
78
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import clear_s3_proxy_config as clear_s3_proxy_config
79
79
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import get_s3_proxy_config as get_s3_proxy_config
80
+ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import get_s3_proxy_config_from_env as get_s3_proxy_config_from_env
80
81
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import get_aws_client as get_aws_client
81
82
  from .mf_extensions.outerbounds.plugins.snowflake.snowflake import Snowflake as Snowflake
82
83
  from .mf_extensions.outerbounds.plugins.checkpoint_datastores.nebius import nebius_checkpoints as nebius_checkpoints
@@ -168,232 +169,21 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
168
169
  ...
169
170
 
170
171
  @typing.overload
171
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
172
- """
173
- Decorator prototype for all step decorators. This function gets specialized
174
- and imported for all decorators types by _import_plugin_decorators().
175
- """
176
- ...
177
-
178
- @typing.overload
179
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
180
- ...
181
-
182
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
183
- """
184
- Decorator prototype for all step decorators. This function gets specialized
185
- and imported for all decorators types by _import_plugin_decorators().
186
- """
187
- ...
188
-
189
- @typing.overload
190
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
191
- """
192
- Specifies a timeout for your step.
193
-
194
- This decorator is useful if this step may hang indefinitely.
195
-
196
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
197
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
198
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
199
-
200
- Note that all the values specified in parameters are added together so if you specify
201
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
202
-
203
-
204
- Parameters
205
- ----------
206
- seconds : int, default 0
207
- Number of seconds to wait prior to timing out.
208
- minutes : int, default 0
209
- Number of minutes to wait prior to timing out.
210
- hours : int, default 0
211
- Number of hours to wait prior to timing out.
212
- """
213
- ...
214
-
215
- @typing.overload
216
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
217
- ...
218
-
219
- @typing.overload
220
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
221
- ...
222
-
223
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
224
- """
225
- Specifies a timeout for your step.
226
-
227
- This decorator is useful if this step may hang indefinitely.
228
-
229
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
230
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
231
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
232
-
233
- Note that all the values specified in parameters are added together so if you specify
234
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
235
-
236
-
237
- Parameters
238
- ----------
239
- seconds : int, default 0
240
- Number of seconds to wait prior to timing out.
241
- minutes : int, default 0
242
- Number of minutes to wait prior to timing out.
243
- hours : int, default 0
244
- Number of hours to wait prior to timing out.
245
- """
246
- ...
247
-
248
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
249
- """
250
- Specifies that this step should execute on DGX cloud.
251
-
252
-
253
- Parameters
254
- ----------
255
- gpu : int
256
- Number of GPUs to use.
257
- gpu_type : str
258
- Type of Nvidia GPU to use.
259
- """
260
- ...
261
-
262
- @typing.overload
263
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
264
- """
265
- Specifies the Conda environment for the step.
266
-
267
- Information in this decorator will augment any
268
- attributes set in the `@conda_base` flow-level decorator. Hence,
269
- you can use `@conda_base` to set packages required by all
270
- steps and use `@conda` to specify step-specific overrides.
271
-
272
-
273
- Parameters
274
- ----------
275
- packages : Dict[str, str], default {}
276
- Packages to use for this step. The key is the name of the package
277
- and the value is the version to use.
278
- libraries : Dict[str, str], default {}
279
- Supported for backward compatibility. When used with packages, packages will take precedence.
280
- python : str, optional, default None
281
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
282
- that the version used will correspond to the version of the Python interpreter used to start the run.
283
- disabled : bool, default False
284
- If set to True, disables @conda.
285
- """
286
- ...
287
-
288
- @typing.overload
289
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
290
- ...
291
-
292
- @typing.overload
293
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
294
- ...
295
-
296
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
297
- """
298
- Specifies the Conda environment for the step.
299
-
300
- Information in this decorator will augment any
301
- attributes set in the `@conda_base` flow-level decorator. Hence,
302
- you can use `@conda_base` to set packages required by all
303
- steps and use `@conda` to specify step-specific overrides.
304
-
305
-
306
- Parameters
307
- ----------
308
- packages : Dict[str, str], default {}
309
- Packages to use for this step. The key is the name of the package
310
- and the value is the version to use.
311
- libraries : Dict[str, str], default {}
312
- Supported for backward compatibility. When used with packages, packages will take precedence.
313
- python : str, optional, default None
314
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
315
- that the version used will correspond to the version of the Python interpreter used to start the run.
316
- disabled : bool, default False
317
- If set to True, disables @conda.
318
- """
319
- ...
320
-
321
- @typing.overload
322
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
172
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
323
173
  """
324
- Specifies the resources needed when executing this step.
325
-
326
- Use `@resources` to specify the resource requirements
327
- independently of the specific compute layer (`@batch`, `@kubernetes`).
328
-
329
- You can choose the compute layer on the command line by executing e.g.
330
- ```
331
- python myflow.py run --with batch
332
- ```
333
- or
334
- ```
335
- python myflow.py run --with kubernetes
336
- ```
337
- which executes the flow on the desired system using the
338
- requirements specified in `@resources`.
339
-
340
-
341
- Parameters
342
- ----------
343
- cpu : int, default 1
344
- Number of CPUs required for this step.
345
- gpu : int, optional, default None
346
- Number of GPUs required for this step.
347
- disk : int, optional, default None
348
- Disk size (in MB) required for this step. Only applies on Kubernetes.
349
- memory : int, default 4096
350
- Memory size (in MB) required for this step.
351
- shared_memory : int, optional, default None
352
- The value for the size (in MiB) of the /dev/shm volume for this step.
353
- This parameter maps to the `--shm-size` option in Docker.
174
+ A simple decorator that demonstrates using CardDecoratorInjector
175
+ to inject a card and render simple markdown content.
354
176
  """
355
177
  ...
356
178
 
357
179
  @typing.overload
358
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
359
- ...
360
-
361
- @typing.overload
362
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
180
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
363
181
  ...
364
182
 
365
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
183
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
366
184
  """
367
- Specifies the resources needed when executing this step.
368
-
369
- Use `@resources` to specify the resource requirements
370
- independently of the specific compute layer (`@batch`, `@kubernetes`).
371
-
372
- You can choose the compute layer on the command line by executing e.g.
373
- ```
374
- python myflow.py run --with batch
375
- ```
376
- or
377
- ```
378
- python myflow.py run --with kubernetes
379
- ```
380
- which executes the flow on the desired system using the
381
- requirements specified in `@resources`.
382
-
383
-
384
- Parameters
385
- ----------
386
- cpu : int, default 1
387
- Number of CPUs required for this step.
388
- gpu : int, optional, default None
389
- Number of GPUs required for this step.
390
- disk : int, optional, default None
391
- Disk size (in MB) required for this step. Only applies on Kubernetes.
392
- memory : int, default 4096
393
- Memory size (in MB) required for this step.
394
- shared_memory : int, optional, default None
395
- The value for the size (in MiB) of the /dev/shm volume for this step.
396
- This parameter maps to the `--shm-size` option in Docker.
185
+ A simple decorator that demonstrates using CardDecoratorInjector
186
+ to inject a card and render simple markdown content.
397
187
  """
398
188
  ...
399
189
 
@@ -527,38 +317,42 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
527
317
  ...
528
318
 
529
319
  @typing.overload
530
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
320
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
531
321
  """
532
- Specifies that the step will success under all circumstances.
533
-
534
- The decorator will create an optional artifact, specified by `var`, which
535
- contains the exception raised. You can use it to detect the presence
536
- of errors, indicating that all happy-path artifacts produced by the step
537
- are missing.
322
+ Specifies environment variables to be set prior to the execution of a step.
538
323
 
539
324
 
540
325
  Parameters
541
326
  ----------
542
- var : str, optional, default None
543
- Name of the artifact in which to store the caught exception.
544
- If not specified, the exception is not stored.
545
- print_exception : bool, default True
546
- Determines whether or not the exception is printed to
547
- stdout when caught.
327
+ vars : Dict[str, str], default {}
328
+ Dictionary of environment variables to set.
548
329
  """
549
330
  ...
550
331
 
551
332
  @typing.overload
552
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
333
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
553
334
  ...
554
335
 
555
336
  @typing.overload
556
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
337
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
557
338
  ...
558
339
 
559
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
340
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
560
341
  """
561
- Specifies that the step will success under all circumstances.
342
+ Specifies environment variables to be set prior to the execution of a step.
343
+
344
+
345
+ Parameters
346
+ ----------
347
+ vars : Dict[str, str], default {}
348
+ Dictionary of environment variables to set.
349
+ """
350
+ ...
351
+
352
+ @typing.overload
353
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
354
+ """
355
+ Specifies that the step will success under all circumstances.
562
356
 
563
357
  The decorator will create an optional artifact, specified by `var`, which
564
358
  contains the exception raised. You can use it to detect the presence
@@ -578,97 +372,102 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
578
372
  ...
579
373
 
580
374
  @typing.overload
581
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
582
- """
583
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
584
- It exists to make it easier for users to know that this decorator should only be used with
585
- a Neo Cloud like CoreWeave.
586
- """
587
- ...
588
-
589
- @typing.overload
590
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
591
- ...
592
-
593
- def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
594
- """
595
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
596
- It exists to make it easier for users to know that this decorator should only be used with
597
- a Neo Cloud like CoreWeave.
598
- """
599
- ...
600
-
601
- @typing.overload
602
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
603
- """
604
- Decorator prototype for all step decorators. This function gets specialized
605
- and imported for all decorators types by _import_plugin_decorators().
606
- """
375
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
607
376
  ...
608
377
 
609
378
  @typing.overload
610
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
379
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
611
380
  ...
612
381
 
613
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
382
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
614
383
  """
615
- Decorator prototype for all step decorators. This function gets specialized
616
- and imported for all decorators types by _import_plugin_decorators().
384
+ Specifies that the step will success under all circumstances.
385
+
386
+ The decorator will create an optional artifact, specified by `var`, which
387
+ contains the exception raised. You can use it to detect the presence
388
+ of errors, indicating that all happy-path artifacts produced by the step
389
+ are missing.
390
+
391
+
392
+ Parameters
393
+ ----------
394
+ var : str, optional, default None
395
+ Name of the artifact in which to store the caught exception.
396
+ If not specified, the exception is not stored.
397
+ print_exception : bool, default True
398
+ Determines whether or not the exception is printed to
399
+ stdout when caught.
617
400
  """
618
401
  ...
619
402
 
620
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
403
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
621
404
  """
622
- S3 Proxy decorator for routing S3 requests through a local proxy service.
405
+ Specifies that this step should execute on DGX cloud.
623
406
 
624
407
 
625
408
  Parameters
626
409
  ----------
627
- integration_name : str, optional
628
- Name of the S3 proxy integration. If not specified, will use the only
629
- available S3 proxy integration in the namespace (fails if multiple exist).
630
- write_mode : str, optional
631
- The desired behavior during write operations to target (origin) S3 bucket.
632
- allowed options are:
633
- "origin-and-cache" -> write to both the target S3 bucket and local object
634
- storage
635
- "origin" -> only write to the target S3 bucket
636
- "cache" -> only write to the object storage service used for caching
637
- debug : bool, optional
638
- Enable debug logging for proxy operations.
410
+ gpu : int
411
+ Number of GPUs to use.
412
+ gpu_type : str
413
+ Type of Nvidia GPU to use.
414
+ queue_timeout : int
415
+ Time to keep the job in NVCF's queue.
639
416
  """
640
417
  ...
641
418
 
642
419
  @typing.overload
643
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
420
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
644
421
  """
645
- Specifies environment variables to be set prior to the execution of a step.
422
+ Specifies the number of times the task corresponding
423
+ to a step needs to be retried.
424
+
425
+ This decorator is useful for handling transient errors, such as networking issues.
426
+ If your task contains operations that can't be retried safely, e.g. database updates,
427
+ it is advisable to annotate it with `@retry(times=0)`.
428
+
429
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
430
+ decorator will execute a no-op task after all retries have been exhausted,
431
+ ensuring that the flow execution can continue.
646
432
 
647
433
 
648
434
  Parameters
649
435
  ----------
650
- vars : Dict[str, str], default {}
651
- Dictionary of environment variables to set.
436
+ times : int, default 3
437
+ Number of times to retry this task.
438
+ minutes_between_retries : int, default 2
439
+ Number of minutes between retries.
652
440
  """
653
441
  ...
654
442
 
655
443
  @typing.overload
656
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
444
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
657
445
  ...
658
446
 
659
447
  @typing.overload
660
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
448
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
661
449
  ...
662
450
 
663
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
451
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
664
452
  """
665
- Specifies environment variables to be set prior to the execution of a step.
453
+ Specifies the number of times the task corresponding
454
+ to a step needs to be retried.
455
+
456
+ This decorator is useful for handling transient errors, such as networking issues.
457
+ If your task contains operations that can't be retried safely, e.g. database updates,
458
+ it is advisable to annotate it with `@retry(times=0)`.
459
+
460
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
461
+ decorator will execute a no-op task after all retries have been exhausted,
462
+ ensuring that the flow execution can continue.
666
463
 
667
464
 
668
465
  Parameters
669
466
  ----------
670
- vars : Dict[str, str], default {}
671
- Dictionary of environment variables to set.
467
+ times : int, default 3
468
+ Number of times to retry this task.
469
+ minutes_between_retries : int, default 2
470
+ Number of minutes between retries.
672
471
  """
673
472
  ...
674
473
 
@@ -819,173 +618,285 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
819
618
  """
820
619
  ...
821
620
 
822
- @typing.overload
823
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
621
+ def nebius_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
824
622
  """
825
- Specifies secrets to be retrieved and injected as environment variables prior to
826
- the execution of a step.
623
+ `@nebius_s3_proxy` is a Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
624
+ It exists to make it easier for users to know that this decorator should only be used with
625
+ a Neo Cloud like Nebius. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
626
+
627
+
628
+ Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
629
+ for S3 read and write requests.
630
+
631
+ This decorator requires an integration in the Outerbounds platform that
632
+ points to an external bucket. It affects S3 operations performed via
633
+ Metaflow's `get_aws_client` and `S3` within a `@step`.
634
+
635
+ Read operations
636
+ ---------------
637
+ All read operations pass through the proxy. If an object does not already
638
+ exist in the external bucket, it is cached there. For example, if code reads
639
+ from buckets `FOO` and `BAR` using the `S3` interface, objects from both
640
+ buckets are cached in the external bucket.
641
+
642
+ During task execution, all S3‑related read requests are routed through the
643
+ proxy:
644
+ - If the object is present in the external object store, the proxy
645
+ streams it directly from there without accessing the requested origin
646
+ bucket.
647
+ - If the object is not present in the external storage, the proxy
648
+ fetches it from the requested bucket, caches it in the external
649
+ storage, and streams the response from the origin bucket.
650
+
651
+ Warning
652
+ -------
653
+ All READ operations (e.g., GetObject, HeadObject) pass through the external
654
+ bucket regardless of the bucket specified in user code. Even
655
+ `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
656
+ external bucket cache.
657
+
658
+ Write operations
659
+ ----------------
660
+ Write behavior is controlled by the `write_mode` parameter, which determines
661
+ whether writes also persist objects in the cache.
662
+
663
+ `write_mode` values:
664
+ - `origin-and-cache`: objects are written both to the cache and to their
665
+ intended origin bucket.
666
+ - `origin`: objects are written only to their intended origin bucket.
827
667
 
828
668
 
829
669
  Parameters
830
670
  ----------
831
- sources : List[Union[str, Dict[str, Any]]], default: []
832
- List of secret specs, defining how the secrets are to be retrieved
833
- role : str, optional, default: None
834
- Role to use for fetching secrets
671
+ integration_name : str, optional
672
+ [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
673
+ that holds the configuration for the external, S3‑compatible object
674
+ storage bucket. If not specified, the only available S3 proxy
675
+ integration in the namespace is used (fails if multiple exist).
676
+ write_mode : str, optional
677
+ Controls whether writes also go to the external bucket.
678
+ - `origin` (default)
679
+ - `origin-and-cache`
680
+ debug : bool, optional
681
+ Enables debug logging for proxy operations.
835
682
  """
836
683
  ...
837
684
 
838
- @typing.overload
839
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
840
- ...
841
-
842
- @typing.overload
843
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
844
- ...
845
-
846
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
685
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
847
686
  """
848
- Specifies secrets to be retrieved and injected as environment variables prior to
849
- the execution of a step.
687
+ Specifies that this step should execute on Kubernetes.
850
688
 
851
689
 
852
690
  Parameters
853
691
  ----------
854
- sources : List[Union[str, Dict[str, Any]]], default: []
855
- List of secret specs, defining how the secrets are to be retrieved
856
- role : str, optional, default: None
857
- Role to use for fetching secrets
692
+ cpu : int, default 1
693
+ Number of CPUs required for this step. If `@resources` is
694
+ also present, the maximum value from all decorators is used.
695
+ memory : int, default 4096
696
+ Memory size (in MB) required for this step. If
697
+ `@resources` is also present, the maximum value from all decorators is
698
+ used.
699
+ disk : int, default 10240
700
+ Disk size (in MB) required for this step. If
701
+ `@resources` is also present, the maximum value from all decorators is
702
+ used.
703
+ image : str, optional, default None
704
+ Docker image to use when launching on Kubernetes. If not specified, and
705
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
706
+ not, a default Docker image mapping to the current version of Python is used.
707
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
708
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
709
+ image_pull_secrets: List[str], default []
710
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
711
+ Kubernetes image pull secrets to use when pulling container images
712
+ in Kubernetes.
713
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
714
+ Kubernetes service account to use when launching pod in Kubernetes.
715
+ secrets : List[str], optional, default None
716
+ Kubernetes secrets to use when launching pod in Kubernetes. These
717
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
718
+ in Metaflow configuration.
719
+ node_selector: Union[Dict[str,str], str], optional, default None
720
+ Kubernetes node selector(s) to apply to the pod running the task.
721
+ Can be passed in as a comma separated string of values e.g.
722
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
723
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
724
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
725
+ Kubernetes namespace to use when launching pod in Kubernetes.
726
+ gpu : int, optional, default None
727
+ Number of GPUs required for this step. A value of zero implies that
728
+ the scheduled node should not have GPUs.
729
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
730
+ The vendor of the GPUs to be used for this step.
731
+ tolerations : List[Dict[str,str]], default []
732
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
733
+ Kubernetes tolerations to use when launching pod in Kubernetes.
734
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
735
+ Kubernetes labels to use when launching pod in Kubernetes.
736
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
737
+ Kubernetes annotations to use when launching pod in Kubernetes.
738
+ use_tmpfs : bool, default False
739
+ This enables an explicit tmpfs mount for this step.
740
+ tmpfs_tempdir : bool, default True
741
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
742
+ tmpfs_size : int, optional, default: None
743
+ The value for the size (in MiB) of the tmpfs mount for this step.
744
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
745
+ memory allocated for this step.
746
+ tmpfs_path : str, optional, default /metaflow_temp
747
+ Path to tmpfs mount for this step.
748
+ persistent_volume_claims : Dict[str, str], optional, default None
749
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
750
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
751
+ shared_memory: int, optional
752
+ Shared memory size (in MiB) required for this step
753
+ port: int, optional
754
+ Port number to specify in the Kubernetes job object
755
+ compute_pool : str, optional, default None
756
+ Compute pool to be used for for this step.
757
+ If not specified, any accessible compute pool within the perimeter is used.
758
+ hostname_resolution_timeout: int, default 10 * 60
759
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
760
+ Only applicable when @parallel is used.
761
+ qos: str, default: Burstable
762
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
763
+
764
+ security_context: Dict[str, Any], optional, default None
765
+ Container security context. Applies to the task container. Allows the following keys:
766
+ - privileged: bool, optional, default None
767
+ - allow_privilege_escalation: bool, optional, default None
768
+ - run_as_user: int, optional, default None
769
+ - run_as_group: int, optional, default None
770
+ - run_as_non_root: bool, optional, default None
858
771
  """
859
772
  ...
860
773
 
861
- @typing.overload
862
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
774
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
863
775
  """
864
- Specifies the number of times the task corresponding
865
- to a step needs to be retried.
776
+ Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
777
+ for S3 read and write requests.
778
+
779
+ This decorator requires an integration in the Outerbounds platform that
780
+ points to an external bucket. It affects S3 operations performed via
781
+ Metaflow's `get_aws_client` and `S3` within a `@step`.
782
+
783
+ Read operations
784
+ ---------------
785
+ All read operations pass through the proxy. If an object does not already
786
+ exist in the external bucket, it is cached there. For example, if code reads
787
+ from buckets `FOO` and `BAR` using the `S3` interface, objects from both
788
+ buckets are cached in the external bucket.
789
+
790
+ During task execution, all S3‑related read requests are routed through the
791
+ proxy:
792
+ - If the object is present in the external object store, the proxy
793
+ streams it directly from there without accessing the requested origin
794
+ bucket.
795
+ - If the object is not present in the external storage, the proxy
796
+ fetches it from the requested bucket, caches it in the external
797
+ storage, and streams the response from the origin bucket.
798
+
799
+ Warning
800
+ -------
801
+ All READ operations (e.g., GetObject, HeadObject) pass through the external
802
+ bucket regardless of the bucket specified in user code. Even
803
+ `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
804
+ external bucket cache.
866
805
 
867
- This decorator is useful for handling transient errors, such as networking issues.
868
- If your task contains operations that can't be retried safely, e.g. database updates,
869
- it is advisable to annotate it with `@retry(times=0)`.
806
+ Write operations
807
+ ----------------
808
+ Write behavior is controlled by the `write_mode` parameter, which determines
809
+ whether writes also persist objects in the cache.
870
810
 
871
- This can be used in conjunction with the `@catch` decorator. The `@catch`
872
- decorator will execute a no-op task after all retries have been exhausted,
873
- ensuring that the flow execution can continue.
811
+ `write_mode` values:
812
+ - `origin-and-cache`: objects are written both to the cache and to their
813
+ intended origin bucket.
814
+ - `origin`: objects are written only to their intended origin bucket.
874
815
 
875
816
 
876
817
  Parameters
877
818
  ----------
878
- times : int, default 3
879
- Number of times to retry this task.
880
- minutes_between_retries : int, default 2
881
- Number of minutes between retries.
819
+ integration_name : str, optional
820
+ [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
821
+ that holds the configuration for the external, S3‑compatible object
822
+ storage bucket. If not specified, the only available S3 proxy
823
+ integration in the namespace is used (fails if multiple exist).
824
+ write_mode : str, optional
825
+ Controls whether writes also go to the external bucket.
826
+ - `origin` (default)
827
+ - `origin-and-cache`
828
+ debug : bool, optional
829
+ Enables debug logging for proxy operations.
882
830
  """
883
831
  ...
884
832
 
885
- @typing.overload
886
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
887
- ...
888
-
889
- @typing.overload
890
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
891
- ...
892
-
893
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
894
- """
895
- Specifies the number of times the task corresponding
896
- to a step needs to be retried.
897
-
898
- This decorator is useful for handling transient errors, such as networking issues.
899
- If your task contains operations that can't be retried safely, e.g. database updates,
900
- it is advisable to annotate it with `@retry(times=0)`.
901
-
902
- This can be used in conjunction with the `@catch` decorator. The `@catch`
903
- decorator will execute a no-op task after all retries have been exhausted,
904
- ensuring that the flow execution can continue.
905
-
906
-
907
- Parameters
908
- ----------
909
- times : int, default 3
910
- Number of times to retry this task.
911
- minutes_between_retries : int, default 2
912
- Number of minutes between retries.
913
- """
914
- ...
915
-
916
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
833
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
917
834
  """
918
835
  Decorator that helps cache, version, and store models/datasets from the Hugging Face Hub.
919
836
 
920
- > Examples
921
-
922
- **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
923
- ```python
924
- @huggingface_hub
925
- @step
926
- def pull_model_from_huggingface(self):
927
- # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
928
- # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
929
- # with the same `repo_id` in the backend storage, it will not download the model again. The return
930
- # value of the function is a reference to the model in the backend storage.
931
- # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
932
-
933
- self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
934
- self.llama_model = current.huggingface_hub.snapshot_download(
935
- repo_id=self.model_id,
936
- allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
937
- )
938
- self.next(self.train)
939
- ```
837
+ Examples
838
+ --------
940
839
 
941
- **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
942
840
  ```python
943
- @huggingface_hub
944
- @step
945
- def run_training(self):
946
- # Temporary directory (auto-cleaned on exit)
947
- with current.huggingface_hub.load(
948
- repo_id="google-bert/bert-base-uncased",
949
- allow_patterns=["*.bin"],
950
- ) as local_path:
951
- # Use files under local_path
952
- train_model(local_path)
953
- ...
841
+ # **Usage: creating references to models from the Hugging Face Hub that may be loaded in downstream steps**
842
+ @huggingface_hub
843
+ @step
844
+ def pull_model_from_huggingface(self):
845
+ # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
846
+ # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
847
+ # with the same `repo_id` in the backend storage, it will not download the model again. The return
848
+ # value of the function is a reference to the model in the backend storage.
849
+ # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
850
+
851
+ self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
852
+ self.llama_model = current.huggingface_hub.snapshot_download(
853
+ repo_id=self.model_id,
854
+ allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
855
+ )
856
+ self.next(self.train)
954
857
 
955
- ```
858
+ # **Usage: explicitly loading models at runtime from the Hugging Face Hub or from cache (from Metaflow's datastore)**
859
+ @huggingface_hub
860
+ @step
861
+ def run_training(self):
862
+ # Temporary directory (auto-cleaned on exit)
863
+ with current.huggingface_hub.load(
864
+ repo_id="google-bert/bert-base-uncased",
865
+ allow_patterns=["*.bin"],
866
+ ) as local_path:
867
+ # Use files under local_path
868
+ train_model(local_path)
869
+ ...
956
870
 
957
- **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
958
- ```python
959
- @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
960
- @step
961
- def pull_model_from_huggingface(self):
962
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
963
- ```
871
+ # **Usage: loading models directly from the Hugging Face Hub or from cache (from Metaflow's datastore)**
964
872
 
965
- ```python
966
- @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
967
- @step
968
- def finetune_model(self):
969
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
970
- # path_to_model will be /my-directory
971
- ```
873
+ @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
874
+ @step
875
+ def pull_model_from_huggingface(self):
876
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
972
877
 
973
- ```python
974
- # Takes all the arguments passed to `snapshot_download`
975
- # except for `local_dir`
976
- @huggingface_hub(load=[
977
- {
978
- "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
979
- },
980
- {
981
- "repo_id": "myorg/mistral-lora",
982
- "repo_type": "model",
983
- },
984
- ])
985
- @step
986
- def finetune_model(self):
987
- path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
988
- # path_to_model will be /my-directory
878
+ @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora", "/my-lora-directory")])
879
+ @step
880
+ def finetune_model(self):
881
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
882
+ # path_to_model will be /my-directory
883
+
884
+
885
+ # Takes all the arguments passed to `snapshot_download`
886
+ # except for `local_dir`
887
+ @huggingface_hub(load=[
888
+ {
889
+ "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
890
+ },
891
+ {
892
+ "repo_id": "myorg/mistral-lora",
893
+ "repo_type": "model",
894
+ },
895
+ ])
896
+ @step
897
+ def finetune_model(self):
898
+ path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
899
+ # path_to_model will be /my-directory
989
900
  ```
990
901
 
991
902
 
@@ -996,34 +907,30 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope:
996
907
 
997
908
  cache_scope : str, optional
998
909
  The scope of the cache. Can be `checkpoint` / `flow` / `global`.
999
-
1000
- - `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
1001
- i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
1002
- Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
1003
-
1004
- - `flow`: All repos are cached under the flow, regardless of namespace.
1005
- i.e., the cached path is derived solely from the flow name.
1006
- When to use this mode:
1007
- - Multiple users are executing the same flow and want shared access to the repos cached by the decorator.
1008
- - Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
1009
-
1010
- - `global`: All repos are cached under a globally static path.
1011
- i.e., the base path of the cache is static and all repos are stored under it.
1012
- When to use this mode:
1013
- - All repos from the Hugging Face Hub need to be shared by users across all flow executions.
1014
-
1015
- Each caching scope comes with its own trade-offs:
1016
- - `checkpoint`:
1017
- - Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
1018
- - Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
1019
- - `flow`:
1020
- - Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
1021
- - The blast radius of a bad checkpoint is limited to all runs of a particular flow.
1022
- - It doesn't promote cache reuse across flows.
1023
- - `global`:
1024
- - Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
1025
- - It promotes cache reuse across flows.
1026
- - The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
910
+ - `checkpoint` (default): All repos are stored like objects saved by `@checkpoint`.
911
+ i.e., the cached path is derived from the namespace, flow, step, and Metaflow foreach iteration.
912
+ Any repo downloaded under this scope will only be retrieved from the cache when the step runs under the same namespace in the same flow (at the same foreach index).
913
+
914
+ - `flow`: All repos are cached under the flow, regardless of namespace.
915
+ i.e., the cached path is derived solely from the flow name.
916
+ When to use this mode: (1) Multiple users are executing the same flow and want shared access to the repos cached by the decorator. (2) Multiple versions of a flow are deployed, all needing access to the same repos cached by the decorator.
917
+
918
+ - `global`: All repos are cached under a globally static path.
919
+ i.e., the base path of the cache is static and all repos are stored under it.
920
+ When to use this mode:
921
+ - All repos from the Hugging Face Hub need to be shared by users across all flow executions.
922
+ - Each caching scope comes with its own trade-offs:
923
+ - `checkpoint`:
924
+ - Has explicit control over when caches are populated (controlled by the same flow that has the `@huggingface_hub` decorator) but ends up hitting the Hugging Face Hub more often if there are many users/namespaces/steps.
925
+ - Since objects are written on a `namespace/flow/step` basis, the blast radius of a bad checkpoint is limited to a particular flow in a namespace.
926
+ - `flow`:
927
+ - Has less control over when caches are populated (can be written by any execution instance of a flow from any namespace) but results in more cache hits.
928
+ - The blast radius of a bad checkpoint is limited to all runs of a particular flow.
929
+ - It doesn't promote cache reuse across flows.
930
+ - `global`:
931
+ - Has no control over when caches are populated (can be written by any flow execution) but has the highest cache hit rate.
932
+ - It promotes cache reuse across flows.
933
+ - The blast radius of a bad checkpoint spans every flow that could be using a particular repo.
1027
934
 
1028
935
  load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
1029
936
  The list of repos (models/datasets) to load.
@@ -1040,308 +947,423 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, cache_scope:
1040
947
  """
1041
948
  ...
1042
949
 
1043
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
950
+ @typing.overload
951
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1044
952
  """
1045
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
953
+ Internal decorator to support Fast bakery
954
+ """
955
+ ...
956
+
957
+ @typing.overload
958
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
959
+ ...
960
+
961
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
962
+ """
963
+ Internal decorator to support Fast bakery
964
+ """
965
+ ...
966
+
967
+ def coreweave_s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
968
+ """
969
+ `@coreweave_s3_proxy` is a CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
970
+ It exists to make it easier for users to know that this decorator should only be used with
971
+ a Neo Cloud like CoreWeave. The underlying mechanics of the decorator is the same as the `@s3_proxy`:
1046
972
 
1047
- User code call
1048
- --------------
1049
- @ollama(
1050
- models=[...],
1051
- ...
1052
- )
1053
973
 
1054
- Valid backend options
1055
- ---------------------
1056
- - 'local': Run as a separate process on the local task machine.
1057
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
1058
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
974
+ Set up an S3 proxy that caches objects in an external, S3‑compatible bucket
975
+ for S3 read and write requests.
1059
976
 
1060
- Valid model options
1061
- -------------------
1062
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
977
+ This decorator requires an integration in the Outerbounds platform that
978
+ points to an external bucket. It affects S3 operations performed via
979
+ Metaflow's `get_aws_client` and `S3` within a `@step`.
980
+
981
+ Read operations
982
+ ---------------
983
+ All read operations pass through the proxy. If an object does not already
984
+ exist in the external bucket, it is cached there. For example, if code reads
985
+ from buckets `FOO` and `BAR` using the `S3` interface, objects from both
986
+ buckets are cached in the external bucket.
987
+
988
+ During task execution, all S3‑related read requests are routed through the
989
+ proxy:
990
+ - If the object is present in the external object store, the proxy
991
+ streams it directly from there without accessing the requested origin
992
+ bucket.
993
+ - If the object is not present in the external storage, the proxy
994
+ fetches it from the requested bucket, caches it in the external
995
+ storage, and streams the response from the origin bucket.
996
+
997
+ Warning
998
+ -------
999
+ All READ operations (e.g., GetObject, HeadObject) pass through the external
1000
+ bucket regardless of the bucket specified in user code. Even
1001
+ `S3(run=self)` and `S3(s3root="mybucketfoo")` requests go through the
1002
+ external bucket cache.
1003
+
1004
+ Write operations
1005
+ ----------------
1006
+ Write behavior is controlled by the `write_mode` parameter, which determines
1007
+ whether writes also persist objects in the cache.
1008
+
1009
+ `write_mode` values:
1010
+ - `origin-and-cache`: objects are written both to the cache and to their
1011
+ intended origin bucket.
1012
+ - `origin`: objects are written only to their intended origin bucket.
1063
1013
 
1064
1014
 
1065
1015
  Parameters
1066
1016
  ----------
1067
- models: list[str]
1068
- List of Ollama containers running models in sidecars.
1069
- backend: str
1070
- Determines where and how to run the Ollama process.
1071
- force_pull: bool
1072
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
1073
- cache_update_policy: str
1074
- Cache update policy: "auto", "force", or "never".
1075
- force_cache_update: bool
1076
- Simple override for "force" cache update policy.
1077
- debug: bool
1078
- Whether to turn on verbose debugging logs.
1079
- circuit_breaker_config: dict
1080
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
1081
- timeout_config: dict
1082
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
1017
+ integration_name : str, optional
1018
+ [Outerbounds integration name](https://docs.outerbounds.com/outerbounds/configuring-secrets/#integrations-view)
1019
+ that holds the configuration for the external, S3‑compatible object
1020
+ storage bucket. If not specified, the only available S3 proxy
1021
+ integration in the namespace is used (fails if multiple exist).
1022
+ write_mode : str, optional
1023
+ Controls whether writes also go to the external bucket.
1024
+ - `origin` (default)
1025
+ - `origin-and-cache`
1026
+ debug : bool, optional
1027
+ Enables debug logging for proxy operations.
1083
1028
  """
1084
1029
  ...
1085
1030
 
1086
1031
  @typing.overload
1087
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1032
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1088
1033
  """
1089
- Internal decorator to support Fast bakery
1034
+ Decorator prototype for all step decorators. This function gets specialized
1035
+ and imported for all decorators types by _import_plugin_decorators().
1090
1036
  """
1091
1037
  ...
1092
1038
 
1093
1039
  @typing.overload
1094
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1040
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1095
1041
  ...
1096
1042
 
1097
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1043
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1098
1044
  """
1099
- Internal decorator to support Fast bakery
1045
+ Decorator prototype for all step decorators. This function gets specialized
1046
+ and imported for all decorators types by _import_plugin_decorators().
1100
1047
  """
1101
1048
  ...
1102
1049
 
1103
1050
  @typing.overload
1104
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1051
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1105
1052
  """
1106
- Creates a human-readable report, a Metaflow Card, after this step completes.
1053
+ Specifies the resources needed when executing this step.
1107
1054
 
1108
- Note that you may add multiple `@card` decorators in a step with different parameters.
1055
+ Use `@resources` to specify the resource requirements
1056
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1057
+
1058
+ You can choose the compute layer on the command line by executing e.g.
1059
+ ```
1060
+ python myflow.py run --with batch
1061
+ ```
1062
+ or
1063
+ ```
1064
+ python myflow.py run --with kubernetes
1065
+ ```
1066
+ which executes the flow on the desired system using the
1067
+ requirements specified in `@resources`.
1109
1068
 
1110
1069
 
1111
1070
  Parameters
1112
1071
  ----------
1113
- type : str, default 'default'
1114
- Card type.
1115
- id : str, optional, default None
1116
- If multiple cards are present, use this id to identify this card.
1117
- options : Dict[str, Any], default {}
1118
- Options passed to the card. The contents depend on the card type.
1119
- timeout : int, default 45
1120
- Interrupt reporting if it takes more than this many seconds.
1072
+ cpu : int, default 1
1073
+ Number of CPUs required for this step.
1074
+ gpu : int, optional, default None
1075
+ Number of GPUs required for this step.
1076
+ disk : int, optional, default None
1077
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1078
+ memory : int, default 4096
1079
+ Memory size (in MB) required for this step.
1080
+ shared_memory : int, optional, default None
1081
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1082
+ This parameter maps to the `--shm-size` option in Docker.
1121
1083
  """
1122
1084
  ...
1123
1085
 
1124
1086
  @typing.overload
1125
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1087
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1126
1088
  ...
1127
1089
 
1128
1090
  @typing.overload
1129
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1091
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1130
1092
  ...
1131
1093
 
1132
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1094
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
1133
1095
  """
1134
- Creates a human-readable report, a Metaflow Card, after this step completes.
1096
+ Specifies the resources needed when executing this step.
1135
1097
 
1136
- Note that you may add multiple `@card` decorators in a step with different parameters.
1098
+ Use `@resources` to specify the resource requirements
1099
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
1100
+
1101
+ You can choose the compute layer on the command line by executing e.g.
1102
+ ```
1103
+ python myflow.py run --with batch
1104
+ ```
1105
+ or
1106
+ ```
1107
+ python myflow.py run --with kubernetes
1108
+ ```
1109
+ which executes the flow on the desired system using the
1110
+ requirements specified in `@resources`.
1137
1111
 
1138
1112
 
1139
1113
  Parameters
1140
1114
  ----------
1141
- type : str, default 'default'
1142
- Card type.
1143
- id : str, optional, default None
1144
- If multiple cards are present, use this id to identify this card.
1145
- options : Dict[str, Any], default {}
1146
- Options passed to the card. The contents depend on the card type.
1147
- timeout : int, default 45
1148
- Interrupt reporting if it takes more than this many seconds.
1115
+ cpu : int, default 1
1116
+ Number of CPUs required for this step.
1117
+ gpu : int, optional, default None
1118
+ Number of GPUs required for this step.
1119
+ disk : int, optional, default None
1120
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1121
+ memory : int, default 4096
1122
+ Memory size (in MB) required for this step.
1123
+ shared_memory : int, optional, default None
1124
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1125
+ This parameter maps to the `--shm-size` option in Docker.
1149
1126
  """
1150
1127
  ...
1151
1128
 
1152
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1129
+ @typing.overload
1130
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1153
1131
  """
1154
- Specifies that this step should execute on Kubernetes.
1132
+ Specifies the PyPI packages for the step.
1133
+
1134
+ Information in this decorator will augment any
1135
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1136
+ you can use `@pypi_base` to set packages required by all
1137
+ steps and use `@pypi` to specify step-specific overrides.
1155
1138
 
1156
1139
 
1157
1140
  Parameters
1158
1141
  ----------
1159
- cpu : int, default 1
1160
- Number of CPUs required for this step. If `@resources` is
1161
- also present, the maximum value from all decorators is used.
1162
- memory : int, default 4096
1163
- Memory size (in MB) required for this step. If
1164
- `@resources` is also present, the maximum value from all decorators is
1165
- used.
1166
- disk : int, default 10240
1167
- Disk size (in MB) required for this step. If
1168
- `@resources` is also present, the maximum value from all decorators is
1169
- used.
1170
- image : str, optional, default None
1171
- Docker image to use when launching on Kubernetes. If not specified, and
1172
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1173
- not, a default Docker image mapping to the current version of Python is used.
1174
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1175
- If given, the imagePullPolicy to be applied to the Docker image of the step.
1176
- image_pull_secrets: List[str], default []
1177
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
1178
- Kubernetes image pull secrets to use when pulling container images
1179
- in Kubernetes.
1180
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1181
- Kubernetes service account to use when launching pod in Kubernetes.
1182
- secrets : List[str], optional, default None
1183
- Kubernetes secrets to use when launching pod in Kubernetes. These
1184
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1185
- in Metaflow configuration.
1186
- node_selector: Union[Dict[str,str], str], optional, default None
1187
- Kubernetes node selector(s) to apply to the pod running the task.
1188
- Can be passed in as a comma separated string of values e.g.
1189
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
1190
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
1191
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1192
- Kubernetes namespace to use when launching pod in Kubernetes.
1193
- gpu : int, optional, default None
1194
- Number of GPUs required for this step. A value of zero implies that
1195
- the scheduled node should not have GPUs.
1196
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1197
- The vendor of the GPUs to be used for this step.
1198
- tolerations : List[Dict[str,str]], default []
1199
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1200
- Kubernetes tolerations to use when launching pod in Kubernetes.
1201
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
1202
- Kubernetes labels to use when launching pod in Kubernetes.
1203
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
1204
- Kubernetes annotations to use when launching pod in Kubernetes.
1205
- use_tmpfs : bool, default False
1206
- This enables an explicit tmpfs mount for this step.
1207
- tmpfs_tempdir : bool, default True
1208
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1209
- tmpfs_size : int, optional, default: None
1210
- The value for the size (in MiB) of the tmpfs mount for this step.
1211
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1212
- memory allocated for this step.
1213
- tmpfs_path : str, optional, default /metaflow_temp
1214
- Path to tmpfs mount for this step.
1215
- persistent_volume_claims : Dict[str, str], optional, default None
1216
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1217
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1218
- shared_memory: int, optional
1219
- Shared memory size (in MiB) required for this step
1220
- port: int, optional
1221
- Port number to specify in the Kubernetes job object
1222
- compute_pool : str, optional, default None
1223
- Compute pool to be used for for this step.
1224
- If not specified, any accessible compute pool within the perimeter is used.
1225
- hostname_resolution_timeout: int, default 10 * 60
1226
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
1227
- Only applicable when @parallel is used.
1228
- qos: str, default: Burstable
1229
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
1142
+ packages : Dict[str, str], default: {}
1143
+ Packages to use for this step. The key is the name of the package
1144
+ and the value is the version to use.
1145
+ python : str, optional, default: None
1146
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1147
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1148
+ """
1149
+ ...
1150
+
1151
+ @typing.overload
1152
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1153
+ ...
1154
+
1155
+ @typing.overload
1156
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1157
+ ...
1158
+
1159
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1160
+ """
1161
+ Specifies the PyPI packages for the step.
1230
1162
 
1231
- security_context: Dict[str, Any], optional, default None
1232
- Container security context. Applies to the task container. Allows the following keys:
1233
- - privileged: bool, optional, default None
1234
- - allow_privilege_escalation: bool, optional, default None
1235
- - run_as_user: int, optional, default None
1236
- - run_as_group: int, optional, default None
1237
- - run_as_non_root: bool, optional, default None
1163
+ Information in this decorator will augment any
1164
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1165
+ you can use `@pypi_base` to set packages required by all
1166
+ steps and use `@pypi` to specify step-specific overrides.
1167
+
1168
+
1169
+ Parameters
1170
+ ----------
1171
+ packages : Dict[str, str], default: {}
1172
+ Packages to use for this step. The key is the name of the package
1173
+ and the value is the version to use.
1174
+ python : str, optional, default: None
1175
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1176
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1177
+ """
1178
+ ...
1179
+
1180
+ @typing.overload
1181
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1182
+ """
1183
+ Specifies a timeout for your step.
1184
+
1185
+ This decorator is useful if this step may hang indefinitely.
1186
+
1187
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1188
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1189
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1190
+
1191
+ Note that all the values specified in parameters are added together so if you specify
1192
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1193
+
1194
+
1195
+ Parameters
1196
+ ----------
1197
+ seconds : int, default 0
1198
+ Number of seconds to wait prior to timing out.
1199
+ minutes : int, default 0
1200
+ Number of minutes to wait prior to timing out.
1201
+ hours : int, default 0
1202
+ Number of hours to wait prior to timing out.
1203
+ """
1204
+ ...
1205
+
1206
+ @typing.overload
1207
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1208
+ ...
1209
+
1210
+ @typing.overload
1211
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1212
+ ...
1213
+
1214
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1215
+ """
1216
+ Specifies a timeout for your step.
1217
+
1218
+ This decorator is useful if this step may hang indefinitely.
1219
+
1220
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1221
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1222
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1223
+
1224
+ Note that all the values specified in parameters are added together so if you specify
1225
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1226
+
1227
+
1228
+ Parameters
1229
+ ----------
1230
+ seconds : int, default 0
1231
+ Number of seconds to wait prior to timing out.
1232
+ minutes : int, default 0
1233
+ Number of minutes to wait prior to timing out.
1234
+ hours : int, default 0
1235
+ Number of hours to wait prior to timing out.
1238
1236
  """
1239
1237
  ...
1240
1238
 
1241
- @typing.overload
1242
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1239
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1243
1240
  """
1244
- A simple decorator that demonstrates using CardDecoratorInjector
1245
- to inject a card and render simple markdown content.
1241
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
1242
+
1243
+ User code call
1244
+ --------------
1245
+ @ollama(
1246
+ models=[...],
1247
+ ...
1248
+ )
1249
+
1250
+ Valid backend options
1251
+ ---------------------
1252
+ - 'local': Run as a separate process on the local task machine.
1253
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
1254
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
1255
+
1256
+ Valid model options
1257
+ -------------------
1258
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
1259
+
1260
+
1261
+ Parameters
1262
+ ----------
1263
+ models: list[str]
1264
+ List of Ollama containers running models in sidecars.
1265
+ backend: str
1266
+ Determines where and how to run the Ollama process.
1267
+ force_pull: bool
1268
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
1269
+ cache_update_policy: str
1270
+ Cache update policy: "auto", "force", or "never".
1271
+ force_cache_update: bool
1272
+ Simple override for "force" cache update policy.
1273
+ debug: bool
1274
+ Whether to turn on verbose debugging logs.
1275
+ circuit_breaker_config: dict
1276
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
1277
+ timeout_config: dict
1278
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
1246
1279
  """
1247
1280
  ...
1248
1281
 
1249
1282
  @typing.overload
1250
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1251
- ...
1252
-
1253
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1254
- """
1255
- A simple decorator that demonstrates using CardDecoratorInjector
1256
- to inject a card and render simple markdown content.
1257
- """
1258
- ...
1259
-
1260
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1283
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1261
1284
  """
1262
- Specifies that this step should execute on DGX cloud.
1285
+ Specifies secrets to be retrieved and injected as environment variables prior to
1286
+ the execution of a step.
1263
1287
 
1264
1288
 
1265
1289
  Parameters
1266
1290
  ----------
1267
- gpu : int
1268
- Number of GPUs to use.
1269
- gpu_type : str
1270
- Type of Nvidia GPU to use.
1271
- queue_timeout : int
1272
- Time to keep the job in NVCF's queue.
1291
+ sources : List[Union[str, Dict[str, Any]]], default: []
1292
+ List of secret specs, defining how the secrets are to be retrieved
1293
+ role : str, optional, default: None
1294
+ Role to use for fetching secrets
1273
1295
  """
1274
1296
  ...
1275
1297
 
1276
1298
  @typing.overload
1277
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1278
- """
1279
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1280
- It exists to make it easier for users to know that this decorator should only be used with
1281
- a Neo Cloud like Nebius.
1282
- """
1299
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1283
1300
  ...
1284
1301
 
1285
1302
  @typing.overload
1286
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1303
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1287
1304
  ...
1288
1305
 
1289
- def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1306
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
1290
1307
  """
1291
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1292
- It exists to make it easier for users to know that this decorator should only be used with
1293
- a Neo Cloud like Nebius.
1308
+ Specifies secrets to be retrieved and injected as environment variables prior to
1309
+ the execution of a step.
1310
+
1311
+
1312
+ Parameters
1313
+ ----------
1314
+ sources : List[Union[str, Dict[str, Any]]], default: []
1315
+ List of secret specs, defining how the secrets are to be retrieved
1316
+ role : str, optional, default: None
1317
+ Role to use for fetching secrets
1294
1318
  """
1295
1319
  ...
1296
1320
 
1297
1321
  @typing.overload
1298
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1322
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1299
1323
  """
1300
- Specifies the PyPI packages for the step.
1324
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1301
1325
 
1302
- Information in this decorator will augment any
1303
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1304
- you can use `@pypi_base` to set packages required by all
1305
- steps and use `@pypi` to specify step-specific overrides.
1326
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1306
1327
 
1307
1328
 
1308
1329
  Parameters
1309
1330
  ----------
1310
- packages : Dict[str, str], default: {}
1311
- Packages to use for this step. The key is the name of the package
1312
- and the value is the version to use.
1313
- python : str, optional, default: None
1314
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1315
- that the version used will correspond to the version of the Python interpreter used to start the run.
1331
+ type : str, default 'default'
1332
+ Card type.
1333
+ id : str, optional, default None
1334
+ If multiple cards are present, use this id to identify this card.
1335
+ options : Dict[str, Any], default {}
1336
+ Options passed to the card. The contents depend on the card type.
1337
+ timeout : int, default 45
1338
+ Interrupt reporting if it takes more than this many seconds.
1316
1339
  """
1317
1340
  ...
1318
1341
 
1319
1342
  @typing.overload
1320
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1343
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1321
1344
  ...
1322
1345
 
1323
1346
  @typing.overload
1324
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1347
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1325
1348
  ...
1326
1349
 
1327
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1350
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1328
1351
  """
1329
- Specifies the PyPI packages for the step.
1352
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1330
1353
 
1331
- Information in this decorator will augment any
1332
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1333
- you can use `@pypi_base` to set packages required by all
1334
- steps and use `@pypi` to specify step-specific overrides.
1354
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1335
1355
 
1336
1356
 
1337
1357
  Parameters
1338
1358
  ----------
1339
- packages : Dict[str, str], default: {}
1340
- Packages to use for this step. The key is the name of the package
1341
- and the value is the version to use.
1342
- python : str, optional, default: None
1343
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1344
- that the version used will correspond to the version of the Python interpreter used to start the run.
1359
+ type : str, default 'default'
1360
+ Card type.
1361
+ id : str, optional, default None
1362
+ If multiple cards are present, use this id to identify this card.
1363
+ options : Dict[str, Any], default {}
1364
+ Options passed to the card. The contents depend on the card type.
1365
+ timeout : int, default 45
1366
+ Interrupt reporting if it takes more than this many seconds.
1345
1367
  """
1346
1368
  ...
1347
1369
 
@@ -1395,97 +1417,181 @@ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card
1395
1417
  """
1396
1418
  ...
1397
1419
 
1398
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1420
+ @typing.overload
1421
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1399
1422
  """
1400
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1401
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1402
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1403
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1404
- starts only after all sensors finish.
1423
+ Specifies the Conda environment for the step.
1424
+
1425
+ Information in this decorator will augment any
1426
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1427
+ you can use `@conda_base` to set packages required by all
1428
+ steps and use `@conda` to specify step-specific overrides.
1429
+
1430
+
1431
+ Parameters
1432
+ ----------
1433
+ packages : Dict[str, str], default {}
1434
+ Packages to use for this step. The key is the name of the package
1435
+ and the value is the version to use.
1436
+ libraries : Dict[str, str], default {}
1437
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1438
+ python : str, optional, default None
1439
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1440
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1441
+ disabled : bool, default False
1442
+ If set to True, disables @conda.
1443
+ """
1444
+ ...
1445
+
1446
+ @typing.overload
1447
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1448
+ ...
1449
+
1450
+ @typing.overload
1451
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1452
+ ...
1453
+
1454
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1455
+ """
1456
+ Specifies the Conda environment for the step.
1457
+
1458
+ Information in this decorator will augment any
1459
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1460
+ you can use `@conda_base` to set packages required by all
1461
+ steps and use `@conda` to specify step-specific overrides.
1462
+
1463
+
1464
+ Parameters
1465
+ ----------
1466
+ packages : Dict[str, str], default {}
1467
+ Packages to use for this step. The key is the name of the package
1468
+ and the value is the version to use.
1469
+ libraries : Dict[str, str], default {}
1470
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1471
+ python : str, optional, default None
1472
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1473
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1474
+ disabled : bool, default False
1475
+ If set to True, disables @conda.
1476
+ """
1477
+ ...
1478
+
1479
+ @typing.overload
1480
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1481
+ """
1482
+ Decorator prototype for all step decorators. This function gets specialized
1483
+ and imported for all decorators types by _import_plugin_decorators().
1484
+ """
1485
+ ...
1486
+
1487
+ @typing.overload
1488
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1489
+ ...
1490
+
1491
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1492
+ """
1493
+ Decorator prototype for all step decorators. This function gets specialized
1494
+ and imported for all decorators types by _import_plugin_decorators().
1495
+ """
1496
+ ...
1497
+
1498
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1499
+ """
1500
+ Specifies that this step should execute on DGX cloud.
1501
+
1502
+
1503
+ Parameters
1504
+ ----------
1505
+ gpu : int
1506
+ Number of GPUs to use.
1507
+ gpu_type : str
1508
+ Type of Nvidia GPU to use.
1509
+ """
1510
+ ...
1511
+
1512
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1513
+ """
1514
+ Specifies what flows belong to the same project.
1515
+
1516
+ A project-specific namespace is created for all flows that
1517
+ use the same `@project(name)`.
1405
1518
 
1406
1519
 
1407
1520
  Parameters
1408
1521
  ----------
1409
- timeout : int
1410
- Time, in seconds before the task times out and fails. (Default: 3600)
1411
- poke_interval : int
1412
- Time in seconds that the job should wait in between each try. (Default: 60)
1413
- mode : str
1414
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1415
- exponential_backoff : bool
1416
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1417
- pool : str
1418
- the slot pool this task should run in,
1419
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1420
- soft_fail : bool
1421
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1422
1522
  name : str
1423
- Name of the sensor on Airflow
1424
- description : str
1425
- Description of sensor in the Airflow UI
1426
- bucket_key : Union[str, List[str]]
1427
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1428
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1429
- bucket_name : str
1430
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1431
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1432
- wildcard_match : bool
1433
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1434
- aws_conn_id : str
1435
- a reference to the s3 connection on Airflow. (Default: None)
1436
- verify : bool
1437
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1523
+ Project name. Make sure that the name is unique amongst all
1524
+ projects that use the same production scheduler. The name may
1525
+ contain only lowercase alphanumeric characters and underscores.
1526
+
1527
+ branch : Optional[str], default None
1528
+ The branch to use. If not specified, the branch is set to
1529
+ `user.<username>` unless `production` is set to `True`. This can
1530
+ also be set on the command line using `--branch` as a top-level option.
1531
+ It is an error to specify `branch` in the decorator and on the command line.
1532
+
1533
+ production : bool, default False
1534
+ Whether or not the branch is the production branch. This can also be set on the
1535
+ command line using `--production` as a top-level option. It is an error to specify
1536
+ `production` in the decorator and on the command line.
1537
+ The project branch name will be:
1538
+ - if `branch` is specified:
1539
+ - if `production` is True: `prod.<branch>`
1540
+ - if `production` is False: `test.<branch>`
1541
+ - if `branch` is not specified:
1542
+ - if `production` is True: `prod`
1543
+ - if `production` is False: `user.<username>`
1438
1544
  """
1439
1545
  ...
1440
1546
 
1441
1547
  @typing.overload
1442
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1548
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1443
1549
  """
1444
- Specifies the times when the flow should be run when running on a
1445
- production scheduler.
1550
+ Specifies the Conda environment for all steps of the flow.
1551
+
1552
+ Use `@conda_base` to set common libraries required by all
1553
+ steps and use `@conda` to specify step-specific additions.
1446
1554
 
1447
1555
 
1448
1556
  Parameters
1449
1557
  ----------
1450
- hourly : bool, default False
1451
- Run the workflow hourly.
1452
- daily : bool, default True
1453
- Run the workflow daily.
1454
- weekly : bool, default False
1455
- Run the workflow weekly.
1456
- cron : str, optional, default None
1457
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1458
- specified by this expression.
1459
- timezone : str, optional, default None
1460
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1461
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1558
+ packages : Dict[str, str], default {}
1559
+ Packages to use for this flow. The key is the name of the package
1560
+ and the value is the version to use.
1561
+ libraries : Dict[str, str], default {}
1562
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1563
+ python : str, optional, default None
1564
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1565
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1566
+ disabled : bool, default False
1567
+ If set to True, disables Conda.
1462
1568
  """
1463
1569
  ...
1464
1570
 
1465
1571
  @typing.overload
1466
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1572
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1467
1573
  ...
1468
1574
 
1469
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1575
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1470
1576
  """
1471
- Specifies the times when the flow should be run when running on a
1472
- production scheduler.
1577
+ Specifies the Conda environment for all steps of the flow.
1578
+
1579
+ Use `@conda_base` to set common libraries required by all
1580
+ steps and use `@conda` to specify step-specific additions.
1473
1581
 
1474
1582
 
1475
1583
  Parameters
1476
1584
  ----------
1477
- hourly : bool, default False
1478
- Run the workflow hourly.
1479
- daily : bool, default True
1480
- Run the workflow daily.
1481
- weekly : bool, default False
1482
- Run the workflow weekly.
1483
- cron : str, optional, default None
1484
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1485
- specified by this expression.
1486
- timezone : str, optional, default None
1487
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1488
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1585
+ packages : Dict[str, str], default {}
1586
+ Packages to use for this flow. The key is the name of the package
1587
+ and the value is the version to use.
1588
+ libraries : Dict[str, str], default {}
1589
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1590
+ python : str, optional, default None
1591
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1592
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1593
+ disabled : bool, default False
1594
+ If set to True, disables Conda.
1489
1595
  """
1490
1596
  ...
1491
1597
 
@@ -1530,6 +1636,142 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
1530
1636
  """
1531
1637
  ...
1532
1638
 
1639
+ @typing.overload
1640
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1641
+ """
1642
+ Specifies the event(s) that this flow depends on.
1643
+
1644
+ ```
1645
+ @trigger(event='foo')
1646
+ ```
1647
+ or
1648
+ ```
1649
+ @trigger(events=['foo', 'bar'])
1650
+ ```
1651
+
1652
+ Additionally, you can specify the parameter mappings
1653
+ to map event payload to Metaflow parameters for the flow.
1654
+ ```
1655
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1656
+ ```
1657
+ or
1658
+ ```
1659
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1660
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1661
+ ```
1662
+
1663
+ 'parameters' can also be a list of strings and tuples like so:
1664
+ ```
1665
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1666
+ ```
1667
+ This is equivalent to:
1668
+ ```
1669
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1670
+ ```
1671
+
1672
+
1673
+ Parameters
1674
+ ----------
1675
+ event : Union[str, Dict[str, Any]], optional, default None
1676
+ Event dependency for this flow.
1677
+ events : List[Union[str, Dict[str, Any]]], default []
1678
+ Events dependency for this flow.
1679
+ options : Dict[str, Any], default {}
1680
+ Backend-specific configuration for tuning eventing behavior.
1681
+ """
1682
+ ...
1683
+
1684
+ @typing.overload
1685
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1686
+ ...
1687
+
1688
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1689
+ """
1690
+ Specifies the event(s) that this flow depends on.
1691
+
1692
+ ```
1693
+ @trigger(event='foo')
1694
+ ```
1695
+ or
1696
+ ```
1697
+ @trigger(events=['foo', 'bar'])
1698
+ ```
1699
+
1700
+ Additionally, you can specify the parameter mappings
1701
+ to map event payload to Metaflow parameters for the flow.
1702
+ ```
1703
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1704
+ ```
1705
+ or
1706
+ ```
1707
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1708
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1709
+ ```
1710
+
1711
+ 'parameters' can also be a list of strings and tuples like so:
1712
+ ```
1713
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1714
+ ```
1715
+ This is equivalent to:
1716
+ ```
1717
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1718
+ ```
1719
+
1720
+
1721
+ Parameters
1722
+ ----------
1723
+ event : Union[str, Dict[str, Any]], optional, default None
1724
+ Event dependency for this flow.
1725
+ events : List[Union[str, Dict[str, Any]]], default []
1726
+ Events dependency for this flow.
1727
+ options : Dict[str, Any], default {}
1728
+ Backend-specific configuration for tuning eventing behavior.
1729
+ """
1730
+ ...
1731
+
1732
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1733
+ """
1734
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1735
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1736
+
1737
+
1738
+ Parameters
1739
+ ----------
1740
+ timeout : int
1741
+ Time, in seconds before the task times out and fails. (Default: 3600)
1742
+ poke_interval : int
1743
+ Time in seconds that the job should wait in between each try. (Default: 60)
1744
+ mode : str
1745
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1746
+ exponential_backoff : bool
1747
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1748
+ pool : str
1749
+ the slot pool this task should run in,
1750
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1751
+ soft_fail : bool
1752
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1753
+ name : str
1754
+ Name of the sensor on Airflow
1755
+ description : str
1756
+ Description of sensor in the Airflow UI
1757
+ external_dag_id : str
1758
+ The dag_id that contains the task you want to wait for.
1759
+ external_task_ids : List[str]
1760
+ The list of task_ids that you want to wait for.
1761
+ If None (default value) the sensor waits for the DAG. (Default: None)
1762
+ allowed_states : List[str]
1763
+ Iterable of allowed states, (Default: ['success'])
1764
+ failed_states : List[str]
1765
+ Iterable of failed or dis-allowed states. (Default: None)
1766
+ execution_delta : datetime.timedelta
1767
+ time difference with the previous execution to look at,
1768
+ the default is the same logical date as the current task or DAG. (Default: None)
1769
+ check_existence: bool
1770
+ Set to True to check if the external task exists or check if
1771
+ the DAG to wait for exists. (Default: True)
1772
+ """
1773
+ ...
1774
+
1533
1775
  @typing.overload
1534
1776
  def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1535
1777
  """
@@ -1615,62 +1857,19 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1615
1857
 
1616
1858
  Note that `branch` is typically one of:
1617
1859
  - `prod`
1618
- - `user.bob`
1619
- - `test.my_experiment`
1620
- - `prod.staging`
1621
-
1622
-
1623
- Parameters
1624
- ----------
1625
- flow : Union[str, Dict[str, str]], optional, default None
1626
- Upstream flow dependency for this flow.
1627
- flows : List[Union[str, Dict[str, str]]], default []
1628
- Upstream flow dependencies for this flow.
1629
- options : Dict[str, Any], default {}
1630
- Backend-specific configuration for tuning eventing behavior.
1631
- """
1632
- ...
1633
-
1634
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1635
- """
1636
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1637
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1860
+ - `user.bob`
1861
+ - `test.my_experiment`
1862
+ - `prod.staging`
1638
1863
 
1639
1864
 
1640
1865
  Parameters
1641
1866
  ----------
1642
- timeout : int
1643
- Time, in seconds before the task times out and fails. (Default: 3600)
1644
- poke_interval : int
1645
- Time in seconds that the job should wait in between each try. (Default: 60)
1646
- mode : str
1647
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1648
- exponential_backoff : bool
1649
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1650
- pool : str
1651
- the slot pool this task should run in,
1652
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1653
- soft_fail : bool
1654
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1655
- name : str
1656
- Name of the sensor on Airflow
1657
- description : str
1658
- Description of sensor in the Airflow UI
1659
- external_dag_id : str
1660
- The dag_id that contains the task you want to wait for.
1661
- external_task_ids : List[str]
1662
- The list of task_ids that you want to wait for.
1663
- If None (default value) the sensor waits for the DAG. (Default: None)
1664
- allowed_states : List[str]
1665
- Iterable of allowed states, (Default: ['success'])
1666
- failed_states : List[str]
1667
- Iterable of failed or dis-allowed states. (Default: None)
1668
- execution_delta : datetime.timedelta
1669
- time difference with the previous execution to look at,
1670
- the default is the same logical date as the current task or DAG. (Default: None)
1671
- check_existence: bool
1672
- Set to True to check if the external task exists or check if
1673
- the DAG to wait for exists. (Default: True)
1867
+ flow : Union[str, Dict[str, str]], optional, default None
1868
+ Upstream flow dependency for this flow.
1869
+ flows : List[Union[str, Dict[str, str]]], default []
1870
+ Upstream flow dependencies for this flow.
1871
+ options : Dict[str, Any], default {}
1872
+ Backend-specific configuration for tuning eventing behavior.
1674
1873
  """
1675
1874
  ...
1676
1875
 
@@ -1788,182 +1987,97 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1788
1987
  """
1789
1988
  ...
1790
1989
 
1791
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1990
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1792
1991
  """
1793
- Specifies what flows belong to the same project.
1794
-
1795
- A project-specific namespace is created for all flows that
1796
- use the same `@project(name)`.
1992
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1993
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1994
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1995
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1996
+ starts only after all sensors finish.
1797
1997
 
1798
1998
 
1799
1999
  Parameters
1800
2000
  ----------
2001
+ timeout : int
2002
+ Time, in seconds before the task times out and fails. (Default: 3600)
2003
+ poke_interval : int
2004
+ Time in seconds that the job should wait in between each try. (Default: 60)
2005
+ mode : str
2006
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
2007
+ exponential_backoff : bool
2008
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
2009
+ pool : str
2010
+ the slot pool this task should run in,
2011
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
2012
+ soft_fail : bool
2013
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1801
2014
  name : str
1802
- Project name. Make sure that the name is unique amongst all
1803
- projects that use the same production scheduler. The name may
1804
- contain only lowercase alphanumeric characters and underscores.
1805
-
1806
- branch : Optional[str], default None
1807
- The branch to use. If not specified, the branch is set to
1808
- `user.<username>` unless `production` is set to `True`. This can
1809
- also be set on the command line using `--branch` as a top-level option.
1810
- It is an error to specify `branch` in the decorator and on the command line.
1811
-
1812
- production : bool, default False
1813
- Whether or not the branch is the production branch. This can also be set on the
1814
- command line using `--production` as a top-level option. It is an error to specify
1815
- `production` in the decorator and on the command line.
1816
- The project branch name will be:
1817
- - if `branch` is specified:
1818
- - if `production` is True: `prod.<branch>`
1819
- - if `production` is False: `test.<branch>`
1820
- - if `branch` is not specified:
1821
- - if `production` is True: `prod`
1822
- - if `production` is False: `user.<username>`
1823
- """
1824
- ...
1825
-
1826
- @typing.overload
1827
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1828
- """
1829
- Specifies the Conda environment for all steps of the flow.
1830
-
1831
- Use `@conda_base` to set common libraries required by all
1832
- steps and use `@conda` to specify step-specific additions.
1833
-
1834
-
1835
- Parameters
1836
- ----------
1837
- packages : Dict[str, str], default {}
1838
- Packages to use for this flow. The key is the name of the package
1839
- and the value is the version to use.
1840
- libraries : Dict[str, str], default {}
1841
- Supported for backward compatibility. When used with packages, packages will take precedence.
1842
- python : str, optional, default None
1843
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1844
- that the version used will correspond to the version of the Python interpreter used to start the run.
1845
- disabled : bool, default False
1846
- If set to True, disables Conda.
1847
- """
1848
- ...
1849
-
1850
- @typing.overload
1851
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1852
- ...
1853
-
1854
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1855
- """
1856
- Specifies the Conda environment for all steps of the flow.
1857
-
1858
- Use `@conda_base` to set common libraries required by all
1859
- steps and use `@conda` to specify step-specific additions.
1860
-
1861
-
1862
- Parameters
1863
- ----------
1864
- packages : Dict[str, str], default {}
1865
- Packages to use for this flow. The key is the name of the package
1866
- and the value is the version to use.
1867
- libraries : Dict[str, str], default {}
1868
- Supported for backward compatibility. When used with packages, packages will take precedence.
1869
- python : str, optional, default None
1870
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1871
- that the version used will correspond to the version of the Python interpreter used to start the run.
1872
- disabled : bool, default False
1873
- If set to True, disables Conda.
2015
+ Name of the sensor on Airflow
2016
+ description : str
2017
+ Description of sensor in the Airflow UI
2018
+ bucket_key : Union[str, List[str]]
2019
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
2020
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
2021
+ bucket_name : str
2022
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
2023
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
2024
+ wildcard_match : bool
2025
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
2026
+ aws_conn_id : str
2027
+ a reference to the s3 connection on Airflow. (Default: None)
2028
+ verify : bool
2029
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1874
2030
  """
1875
2031
  ...
1876
2032
 
1877
2033
  @typing.overload
1878
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
2034
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1879
2035
  """
1880
- Specifies the event(s) that this flow depends on.
1881
-
1882
- ```
1883
- @trigger(event='foo')
1884
- ```
1885
- or
1886
- ```
1887
- @trigger(events=['foo', 'bar'])
1888
- ```
1889
-
1890
- Additionally, you can specify the parameter mappings
1891
- to map event payload to Metaflow parameters for the flow.
1892
- ```
1893
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1894
- ```
1895
- or
1896
- ```
1897
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1898
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1899
- ```
1900
-
1901
- 'parameters' can also be a list of strings and tuples like so:
1902
- ```
1903
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1904
- ```
1905
- This is equivalent to:
1906
- ```
1907
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1908
- ```
2036
+ Specifies the times when the flow should be run when running on a
2037
+ production scheduler.
1909
2038
 
1910
2039
 
1911
2040
  Parameters
1912
2041
  ----------
1913
- event : Union[str, Dict[str, Any]], optional, default None
1914
- Event dependency for this flow.
1915
- events : List[Union[str, Dict[str, Any]]], default []
1916
- Events dependency for this flow.
1917
- options : Dict[str, Any], default {}
1918
- Backend-specific configuration for tuning eventing behavior.
2042
+ hourly : bool, default False
2043
+ Run the workflow hourly.
2044
+ daily : bool, default True
2045
+ Run the workflow daily.
2046
+ weekly : bool, default False
2047
+ Run the workflow weekly.
2048
+ cron : str, optional, default None
2049
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
2050
+ specified by this expression.
2051
+ timezone : str, optional, default None
2052
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
2053
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1919
2054
  """
1920
2055
  ...
1921
2056
 
1922
2057
  @typing.overload
1923
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
2058
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1924
2059
  ...
1925
2060
 
1926
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
2061
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1927
2062
  """
1928
- Specifies the event(s) that this flow depends on.
1929
-
1930
- ```
1931
- @trigger(event='foo')
1932
- ```
1933
- or
1934
- ```
1935
- @trigger(events=['foo', 'bar'])
1936
- ```
1937
-
1938
- Additionally, you can specify the parameter mappings
1939
- to map event payload to Metaflow parameters for the flow.
1940
- ```
1941
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1942
- ```
1943
- or
1944
- ```
1945
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1946
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1947
- ```
1948
-
1949
- 'parameters' can also be a list of strings and tuples like so:
1950
- ```
1951
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1952
- ```
1953
- This is equivalent to:
1954
- ```
1955
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1956
- ```
2063
+ Specifies the times when the flow should be run when running on a
2064
+ production scheduler.
1957
2065
 
1958
2066
 
1959
2067
  Parameters
1960
2068
  ----------
1961
- event : Union[str, Dict[str, Any]], optional, default None
1962
- Event dependency for this flow.
1963
- events : List[Union[str, Dict[str, Any]]], default []
1964
- Events dependency for this flow.
1965
- options : Dict[str, Any], default {}
1966
- Backend-specific configuration for tuning eventing behavior.
2069
+ hourly : bool, default False
2070
+ Run the workflow hourly.
2071
+ daily : bool, default True
2072
+ Run the workflow daily.
2073
+ weekly : bool, default False
2074
+ Run the workflow weekly.
2075
+ cron : str, optional, default None
2076
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
2077
+ specified by this expression.
2078
+ timezone : str, optional, default None
2079
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
2080
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1967
2081
  """
1968
2082
  ...
1969
2083