ob-metaflow-stubs 6.0.3.188rc2__py2.py3-none-any.whl → 6.0.3.188rc4__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (243) hide show
  1. metaflow-stubs/__init__.pyi +876 -871
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +3 -3
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +2 -2
  14. metaflow-stubs/info_file.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +1 -1
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +41 -28
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +1 -1
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +1 -1
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +1 -1
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +1 -1
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +9 -1
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +5 -3
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +1 -1
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +1 -1
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +4 -4
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +1 -1
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +1 -1
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +3 -1
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +63 -44
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +1 -1
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +52 -0
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +28 -1
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +2 -2
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +46 -0
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +1 -1
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +1 -1
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +3 -3
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +1 -1
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +1 -1
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +1 -1
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +1 -1
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  110. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  111. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  112. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
  113. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  114. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -1
  115. metaflow-stubs/multicore_utils.pyi +1 -1
  116. metaflow-stubs/ob_internal.pyi +2 -1
  117. metaflow-stubs/parameters.pyi +2 -2
  118. metaflow-stubs/plugins/__init__.pyi +8 -8
  119. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  120. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  121. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  122. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  123. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  124. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  125. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  126. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  127. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  128. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  129. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  130. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  131. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  132. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  133. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  134. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  135. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  136. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  137. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  138. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  139. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  140. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  141. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  142. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  143. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  144. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  145. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  146. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  147. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  148. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  149. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  150. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  151. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  152. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  153. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  154. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  155. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  156. metaflow-stubs/plugins/cards/__init__.pyi +5 -5
  157. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  158. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  159. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  160. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  161. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  162. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  163. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  164. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  165. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  166. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  167. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  168. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  169. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  170. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  171. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  172. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  173. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  174. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  175. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  176. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  177. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  178. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  179. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  180. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  181. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  182. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  183. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  184. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  185. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  186. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  187. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  188. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  189. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  190. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  191. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
  192. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  193. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  194. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  195. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  196. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  197. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  198. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  199. metaflow-stubs/plugins/perimeters.pyi +1 -1
  200. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  201. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  202. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  203. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  204. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  205. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  206. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  207. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  208. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  209. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  210. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  211. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  212. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  213. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  214. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  215. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  216. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  217. metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
  218. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  219. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  220. metaflow-stubs/profilers/__init__.pyi +1 -1
  221. metaflow-stubs/pylint_wrapper.pyi +1 -1
  222. metaflow-stubs/runner/__init__.pyi +1 -1
  223. metaflow-stubs/runner/deployer.pyi +4 -4
  224. metaflow-stubs/runner/deployer_impl.pyi +1 -1
  225. metaflow-stubs/runner/metaflow_runner.pyi +1 -1
  226. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  227. metaflow-stubs/runner/nbrun.pyi +1 -1
  228. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  229. metaflow-stubs/runner/utils.pyi +3 -3
  230. metaflow-stubs/system/__init__.pyi +1 -1
  231. metaflow-stubs/system/system_logger.pyi +2 -2
  232. metaflow-stubs/system/system_monitor.pyi +1 -1
  233. metaflow-stubs/tagging_util.pyi +1 -1
  234. metaflow-stubs/tuple_util.pyi +1 -1
  235. metaflow-stubs/user_configs/__init__.pyi +1 -1
  236. metaflow-stubs/user_configs/config_decorators.pyi +4 -4
  237. metaflow-stubs/user_configs/config_options.pyi +1 -1
  238. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  239. {ob_metaflow_stubs-6.0.3.188rc2.dist-info → ob_metaflow_stubs-6.0.3.188rc4.dist-info}/METADATA +1 -1
  240. ob_metaflow_stubs-6.0.3.188rc4.dist-info/RECORD +243 -0
  241. ob_metaflow_stubs-6.0.3.188rc2.dist-info/RECORD +0 -241
  242. {ob_metaflow_stubs-6.0.3.188rc2.dist-info → ob_metaflow_stubs-6.0.3.188rc4.dist-info}/WHEEL +0 -0
  243. {ob_metaflow_stubs-6.0.3.188rc2.dist-info → ob_metaflow_stubs-6.0.3.188rc4.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.15.18.1+obcheckpoint(0.2.4);ob(v1) #
4
- # Generated on 2025-07-09T05:17:09.660228 #
4
+ # Generated on 2025-07-10T23:14:33.334494 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -37,16 +37,16 @@ from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDec
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
38
  from . import metaflow_git as metaflow_git
39
39
  from . import cards as cards
40
- from . import events as events
41
40
  from . import tuple_util as tuple_util
41
+ from . import events as events
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
48
47
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
49
48
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
49
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
52
52
  from .client.core import get_namespace as get_namespace
@@ -73,6 +73,7 @@ from .mf_extensions.outerbounds.plugins.snowflake.snowflake import Snowflake as
73
73
  from .mf_extensions.outerbounds.plugins.checkpoint_datastores.nebius import nebius_checkpoints as nebius_checkpoints
74
74
  from .mf_extensions.outerbounds.plugins.checkpoint_datastores.coreweave import coreweave_checkpoints as coreweave_checkpoints
75
75
  from .mf_extensions.outerbounds.plugins.aws.assume_role_decorator import assume_role as assume_role
76
+ from .mf_extensions.outerbounds.plugins.apps.core.deployer import AppDeployer as AppDeployer
76
77
  from . import cli_components as cli_components
77
78
  from . import system as system
78
79
  from . import pylint_wrapper as pylint_wrapper
@@ -155,6 +156,154 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
155
156
  """
156
157
  ...
157
158
 
159
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
160
+ """
161
+ Specifies that this step should execute on Kubernetes.
162
+
163
+
164
+ Parameters
165
+ ----------
166
+ cpu : int, default 1
167
+ Number of CPUs required for this step. If `@resources` is
168
+ also present, the maximum value from all decorators is used.
169
+ memory : int, default 4096
170
+ Memory size (in MB) required for this step. If
171
+ `@resources` is also present, the maximum value from all decorators is
172
+ used.
173
+ disk : int, default 10240
174
+ Disk size (in MB) required for this step. If
175
+ `@resources` is also present, the maximum value from all decorators is
176
+ used.
177
+ image : str, optional, default None
178
+ Docker image to use when launching on Kubernetes. If not specified, and
179
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
180
+ not, a default Docker image mapping to the current version of Python is used.
181
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
182
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
183
+ image_pull_secrets: List[str], default []
184
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
185
+ Kubernetes image pull secrets to use when pulling container images
186
+ in Kubernetes.
187
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
188
+ Kubernetes service account to use when launching pod in Kubernetes.
189
+ secrets : List[str], optional, default None
190
+ Kubernetes secrets to use when launching pod in Kubernetes. These
191
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
192
+ in Metaflow configuration.
193
+ node_selector: Union[Dict[str,str], str], optional, default None
194
+ Kubernetes node selector(s) to apply to the pod running the task.
195
+ Can be passed in as a comma separated string of values e.g.
196
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
197
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
198
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
199
+ Kubernetes namespace to use when launching pod in Kubernetes.
200
+ gpu : int, optional, default None
201
+ Number of GPUs required for this step. A value of zero implies that
202
+ the scheduled node should not have GPUs.
203
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
204
+ The vendor of the GPUs to be used for this step.
205
+ tolerations : List[str], default []
206
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
207
+ Kubernetes tolerations to use when launching pod in Kubernetes.
208
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
209
+ Kubernetes labels to use when launching pod in Kubernetes.
210
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
211
+ Kubernetes annotations to use when launching pod in Kubernetes.
212
+ use_tmpfs : bool, default False
213
+ This enables an explicit tmpfs mount for this step.
214
+ tmpfs_tempdir : bool, default True
215
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
216
+ tmpfs_size : int, optional, default: None
217
+ The value for the size (in MiB) of the tmpfs mount for this step.
218
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
219
+ memory allocated for this step.
220
+ tmpfs_path : str, optional, default /metaflow_temp
221
+ Path to tmpfs mount for this step.
222
+ persistent_volume_claims : Dict[str, str], optional, default None
223
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
224
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
225
+ shared_memory: int, optional
226
+ Shared memory size (in MiB) required for this step
227
+ port: int, optional
228
+ Port number to specify in the Kubernetes job object
229
+ compute_pool : str, optional, default None
230
+ Compute pool to be used for for this step.
231
+ If not specified, any accessible compute pool within the perimeter is used.
232
+ hostname_resolution_timeout: int, default 10 * 60
233
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
234
+ Only applicable when @parallel is used.
235
+ qos: str, default: Burstable
236
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
237
+
238
+ security_context: Dict[str, Any], optional, default None
239
+ Container security context. Applies to the task container. Allows the following keys:
240
+ - privileged: bool, optional, default None
241
+ - allow_privilege_escalation: bool, optional, default None
242
+ - run_as_user: int, optional, default None
243
+ - run_as_group: int, optional, default None
244
+ - run_as_non_root: bool, optional, default None
245
+ """
246
+ ...
247
+
248
+ @typing.overload
249
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
250
+ """
251
+ Specifies the Conda environment for the step.
252
+
253
+ Information in this decorator will augment any
254
+ attributes set in the `@conda_base` flow-level decorator. Hence,
255
+ you can use `@conda_base` to set packages required by all
256
+ steps and use `@conda` to specify step-specific overrides.
257
+
258
+
259
+ Parameters
260
+ ----------
261
+ packages : Dict[str, str], default {}
262
+ Packages to use for this step. The key is the name of the package
263
+ and the value is the version to use.
264
+ libraries : Dict[str, str], default {}
265
+ Supported for backward compatibility. When used with packages, packages will take precedence.
266
+ python : str, optional, default None
267
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
268
+ that the version used will correspond to the version of the Python interpreter used to start the run.
269
+ disabled : bool, default False
270
+ If set to True, disables @conda.
271
+ """
272
+ ...
273
+
274
+ @typing.overload
275
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
276
+ ...
277
+
278
+ @typing.overload
279
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
280
+ ...
281
+
282
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
283
+ """
284
+ Specifies the Conda environment for the step.
285
+
286
+ Information in this decorator will augment any
287
+ attributes set in the `@conda_base` flow-level decorator. Hence,
288
+ you can use `@conda_base` to set packages required by all
289
+ steps and use `@conda` to specify step-specific overrides.
290
+
291
+
292
+ Parameters
293
+ ----------
294
+ packages : Dict[str, str], default {}
295
+ Packages to use for this step. The key is the name of the package
296
+ and the value is the version to use.
297
+ libraries : Dict[str, str], default {}
298
+ Supported for backward compatibility. When used with packages, packages will take precedence.
299
+ python : str, optional, default None
300
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
301
+ that the version used will correspond to the version of the Python interpreter used to start the run.
302
+ disabled : bool, default False
303
+ If set to True, disables @conda.
304
+ """
305
+ ...
306
+
158
307
  @typing.overload
159
308
  def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
160
309
  """
@@ -234,186 +383,175 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
234
383
  """
235
384
  ...
236
385
 
237
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
238
- """
239
- Specifies that this step should execute on DGX cloud.
240
-
241
-
242
- Parameters
243
- ----------
244
- gpu : int
245
- Number of GPUs to use.
246
- gpu_type : str
247
- Type of Nvidia GPU to use.
248
- queue_timeout : int
249
- Time to keep the job in NVCF's queue.
250
- """
251
- ...
252
-
253
386
  @typing.overload
254
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
387
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
255
388
  """
256
- Specifies a timeout for your step.
257
-
258
- This decorator is useful if this step may hang indefinitely.
259
-
260
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
261
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
262
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
389
+ Creates a human-readable report, a Metaflow Card, after this step completes.
263
390
 
264
- Note that all the values specified in parameters are added together so if you specify
265
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
391
+ Note that you may add multiple `@card` decorators in a step with different parameters.
266
392
 
267
393
 
268
394
  Parameters
269
395
  ----------
270
- seconds : int, default 0
271
- Number of seconds to wait prior to timing out.
272
- minutes : int, default 0
273
- Number of minutes to wait prior to timing out.
274
- hours : int, default 0
275
- Number of hours to wait prior to timing out.
396
+ type : str, default 'default'
397
+ Card type.
398
+ id : str, optional, default None
399
+ If multiple cards are present, use this id to identify this card.
400
+ options : Dict[str, Any], default {}
401
+ Options passed to the card. The contents depend on the card type.
402
+ timeout : int, default 45
403
+ Interrupt reporting if it takes more than this many seconds.
276
404
  """
277
405
  ...
278
406
 
279
407
  @typing.overload
280
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
408
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
281
409
  ...
282
410
 
283
411
  @typing.overload
284
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
412
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
285
413
  ...
286
414
 
287
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
415
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
288
416
  """
289
- Specifies a timeout for your step.
290
-
291
- This decorator is useful if this step may hang indefinitely.
292
-
293
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
294
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
295
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
417
+ Creates a human-readable report, a Metaflow Card, after this step completes.
296
418
 
297
- Note that all the values specified in parameters are added together so if you specify
298
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
419
+ Note that you may add multiple `@card` decorators in a step with different parameters.
299
420
 
300
421
 
301
422
  Parameters
302
423
  ----------
303
- seconds : int, default 0
304
- Number of seconds to wait prior to timing out.
305
- minutes : int, default 0
306
- Number of minutes to wait prior to timing out.
307
- hours : int, default 0
308
- Number of hours to wait prior to timing out.
424
+ type : str, default 'default'
425
+ Card type.
426
+ id : str, optional, default None
427
+ If multiple cards are present, use this id to identify this card.
428
+ options : Dict[str, Any], default {}
429
+ Options passed to the card. The contents depend on the card type.
430
+ timeout : int, default 45
431
+ Interrupt reporting if it takes more than this many seconds.
309
432
  """
310
433
  ...
311
434
 
312
435
  @typing.overload
313
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
436
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
314
437
  """
315
- Specifies the PyPI packages for the step.
438
+ Specifies that the step will success under all circumstances.
316
439
 
317
- Information in this decorator will augment any
318
- attributes set in the `@pyi_base` flow-level decorator. Hence,
319
- you can use `@pypi_base` to set packages required by all
320
- steps and use `@pypi` to specify step-specific overrides.
440
+ The decorator will create an optional artifact, specified by `var`, which
441
+ contains the exception raised. You can use it to detect the presence
442
+ of errors, indicating that all happy-path artifacts produced by the step
443
+ are missing.
321
444
 
322
445
 
323
446
  Parameters
324
447
  ----------
325
- packages : Dict[str, str], default: {}
326
- Packages to use for this step. The key is the name of the package
327
- and the value is the version to use.
328
- python : str, optional, default: None
329
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
330
- that the version used will correspond to the version of the Python interpreter used to start the run.
448
+ var : str, optional, default None
449
+ Name of the artifact in which to store the caught exception.
450
+ If not specified, the exception is not stored.
451
+ print_exception : bool, default True
452
+ Determines whether or not the exception is printed to
453
+ stdout when caught.
331
454
  """
332
455
  ...
333
456
 
334
457
  @typing.overload
335
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
458
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
336
459
  ...
337
460
 
338
461
  @typing.overload
339
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
462
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
340
463
  ...
341
464
 
342
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
465
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
343
466
  """
344
- Specifies the PyPI packages for the step.
467
+ Specifies that the step will success under all circumstances.
345
468
 
346
- Information in this decorator will augment any
347
- attributes set in the `@pyi_base` flow-level decorator. Hence,
348
- you can use `@pypi_base` to set packages required by all
349
- steps and use `@pypi` to specify step-specific overrides.
469
+ The decorator will create an optional artifact, specified by `var`, which
470
+ contains the exception raised. You can use it to detect the presence
471
+ of errors, indicating that all happy-path artifacts produced by the step
472
+ are missing.
350
473
 
351
474
 
352
475
  Parameters
353
476
  ----------
354
- packages : Dict[str, str], default: {}
355
- Packages to use for this step. The key is the name of the package
356
- and the value is the version to use.
357
- python : str, optional, default: None
358
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
359
- that the version used will correspond to the version of the Python interpreter used to start the run.
477
+ var : str, optional, default None
478
+ Name of the artifact in which to store the caught exception.
479
+ If not specified, the exception is not stored.
480
+ print_exception : bool, default True
481
+ Determines whether or not the exception is printed to
482
+ stdout when caught.
360
483
  """
361
484
  ...
362
485
 
363
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
486
+ @typing.overload
487
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
364
488
  """
365
- Specifies that this step should execute on DGX cloud.
489
+ Internal decorator to support Fast bakery
490
+ """
491
+ ...
492
+
493
+ @typing.overload
494
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
495
+ ...
496
+
497
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
498
+ """
499
+ Internal decorator to support Fast bakery
500
+ """
501
+ ...
502
+
503
+ @typing.overload
504
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
505
+ """
506
+ Specifies the number of times the task corresponding
507
+ to a step needs to be retried.
508
+
509
+ This decorator is useful for handling transient errors, such as networking issues.
510
+ If your task contains operations that can't be retried safely, e.g. database updates,
511
+ it is advisable to annotate it with `@retry(times=0)`.
512
+
513
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
514
+ decorator will execute a no-op task after all retries have been exhausted,
515
+ ensuring that the flow execution can continue.
366
516
 
367
517
 
368
518
  Parameters
369
519
  ----------
370
- gpu : int
371
- Number of GPUs to use.
372
- gpu_type : str
373
- Type of Nvidia GPU to use.
520
+ times : int, default 3
521
+ Number of times to retry this task.
522
+ minutes_between_retries : int, default 2
523
+ Number of minutes between retries.
374
524
  """
375
525
  ...
376
526
 
377
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
527
+ @typing.overload
528
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
529
+ ...
530
+
531
+ @typing.overload
532
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
533
+ ...
534
+
535
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
378
536
  """
379
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
380
-
381
- User code call
382
- --------------
383
- @ollama(
384
- models=[...],
385
- ...
386
- )
537
+ Specifies the number of times the task corresponding
538
+ to a step needs to be retried.
387
539
 
388
- Valid backend options
389
- ---------------------
390
- - 'local': Run as a separate process on the local task machine.
391
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
392
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
540
+ This decorator is useful for handling transient errors, such as networking issues.
541
+ If your task contains operations that can't be retried safely, e.g. database updates,
542
+ it is advisable to annotate it with `@retry(times=0)`.
393
543
 
394
- Valid model options
395
- -------------------
396
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
544
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
545
+ decorator will execute a no-op task after all retries have been exhausted,
546
+ ensuring that the flow execution can continue.
397
547
 
398
548
 
399
549
  Parameters
400
550
  ----------
401
- models: list[str]
402
- List of Ollama containers running models in sidecars.
403
- backend: str
404
- Determines where and how to run the Ollama process.
405
- force_pull: bool
406
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
407
- cache_update_policy: str
408
- Cache update policy: "auto", "force", or "never".
409
- force_cache_update: bool
410
- Simple override for "force" cache update policy.
411
- debug: bool
412
- Whether to turn on verbose debugging logs.
413
- circuit_breaker_config: dict
414
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
415
- timeout_config: dict
416
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
551
+ times : int, default 3
552
+ Number of times to retry this task.
553
+ minutes_between_retries : int, default 2
554
+ Number of minutes between retries.
417
555
  """
418
556
  ...
419
557
 
@@ -467,116 +605,6 @@ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card
467
605
  """
468
606
  ...
469
607
 
470
- @typing.overload
471
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
472
- """
473
- Specifies the Conda environment for the step.
474
-
475
- Information in this decorator will augment any
476
- attributes set in the `@conda_base` flow-level decorator. Hence,
477
- you can use `@conda_base` to set packages required by all
478
- steps and use `@conda` to specify step-specific overrides.
479
-
480
-
481
- Parameters
482
- ----------
483
- packages : Dict[str, str], default {}
484
- Packages to use for this step. The key is the name of the package
485
- and the value is the version to use.
486
- libraries : Dict[str, str], default {}
487
- Supported for backward compatibility. When used with packages, packages will take precedence.
488
- python : str, optional, default None
489
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
490
- that the version used will correspond to the version of the Python interpreter used to start the run.
491
- disabled : bool, default False
492
- If set to True, disables @conda.
493
- """
494
- ...
495
-
496
- @typing.overload
497
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
498
- ...
499
-
500
- @typing.overload
501
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
502
- ...
503
-
504
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
505
- """
506
- Specifies the Conda environment for the step.
507
-
508
- Information in this decorator will augment any
509
- attributes set in the `@conda_base` flow-level decorator. Hence,
510
- you can use `@conda_base` to set packages required by all
511
- steps and use `@conda` to specify step-specific overrides.
512
-
513
-
514
- Parameters
515
- ----------
516
- packages : Dict[str, str], default {}
517
- Packages to use for this step. The key is the name of the package
518
- and the value is the version to use.
519
- libraries : Dict[str, str], default {}
520
- Supported for backward compatibility. When used with packages, packages will take precedence.
521
- python : str, optional, default None
522
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
523
- that the version used will correspond to the version of the Python interpreter used to start the run.
524
- disabled : bool, default False
525
- If set to True, disables @conda.
526
- """
527
- ...
528
-
529
- @typing.overload
530
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
531
- """
532
- Specifies that the step will success under all circumstances.
533
-
534
- The decorator will create an optional artifact, specified by `var`, which
535
- contains the exception raised. You can use it to detect the presence
536
- of errors, indicating that all happy-path artifacts produced by the step
537
- are missing.
538
-
539
-
540
- Parameters
541
- ----------
542
- var : str, optional, default None
543
- Name of the artifact in which to store the caught exception.
544
- If not specified, the exception is not stored.
545
- print_exception : bool, default True
546
- Determines whether or not the exception is printed to
547
- stdout when caught.
548
- """
549
- ...
550
-
551
- @typing.overload
552
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
553
- ...
554
-
555
- @typing.overload
556
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
557
- ...
558
-
559
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
560
- """
561
- Specifies that the step will success under all circumstances.
562
-
563
- The decorator will create an optional artifact, specified by `var`, which
564
- contains the exception raised. You can use it to detect the presence
565
- of errors, indicating that all happy-path artifacts produced by the step
566
- are missing.
567
-
568
-
569
- Parameters
570
- ----------
571
- var : str, optional, default None
572
- Name of the artifact in which to store the caught exception.
573
- If not specified, the exception is not stored.
574
- print_exception : bool, default True
575
- Determines whether or not the exception is printed to
576
- stdout when caught.
577
- """
578
- ...
579
-
580
608
  def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
581
609
  """
582
610
  Decorator that helps cache, version and store models/datasets from huggingface hub.
@@ -658,267 +686,16 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
658
686
  ...
659
687
 
660
688
  @typing.overload
661
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
689
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
662
690
  """
663
- Enables loading / saving of models within a step.
691
+ Enables checkpointing for a step.
664
692
 
665
693
  > Examples
666
- - Saving Models
667
- ```python
668
- @model
669
- @step
670
- def train(self):
671
- # current.model.save returns a dictionary reference to the model saved
672
- self.my_model = current.model.save(
673
- path_to_my_model,
674
- label="my_model",
675
- metadata={
676
- "epochs": 10,
677
- "batch-size": 32,
678
- "learning-rate": 0.001,
679
- }
680
- )
681
- self.next(self.test)
682
694
 
683
- @model(load="my_model")
684
- @step
685
- def test(self):
686
- # `current.model.loaded` returns a dictionary of the loaded models
687
- # where the key is the name of the artifact and the value is the path to the model
688
- print(os.listdir(current.model.loaded["my_model"]))
689
- self.next(self.end)
690
- ```
695
+ - Saving Checkpoints
691
696
 
692
- - Loading models
693
697
  ```python
694
- @step
695
- def train(self):
696
- # current.model.load returns the path to the model loaded
697
- checkpoint_path = current.model.load(
698
- self.checkpoint_key,
699
- )
700
- model_path = current.model.load(
701
- self.model,
702
- )
703
- self.next(self.test)
704
- ```
705
-
706
-
707
- Parameters
708
- ----------
709
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
710
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
711
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
712
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
713
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
714
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
715
-
716
- temp_dir_root : str, default: None
717
- The root directory under which `current.model.loaded` will store loaded models
718
- """
719
- ...
720
-
721
- @typing.overload
722
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
723
- ...
724
-
725
- @typing.overload
726
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
727
- ...
728
-
729
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
730
- """
731
- Enables loading / saving of models within a step.
732
-
733
- > Examples
734
- - Saving Models
735
- ```python
736
- @model
737
- @step
738
- def train(self):
739
- # current.model.save returns a dictionary reference to the model saved
740
- self.my_model = current.model.save(
741
- path_to_my_model,
742
- label="my_model",
743
- metadata={
744
- "epochs": 10,
745
- "batch-size": 32,
746
- "learning-rate": 0.001,
747
- }
748
- )
749
- self.next(self.test)
750
-
751
- @model(load="my_model")
752
- @step
753
- def test(self):
754
- # `current.model.loaded` returns a dictionary of the loaded models
755
- # where the key is the name of the artifact and the value is the path to the model
756
- print(os.listdir(current.model.loaded["my_model"]))
757
- self.next(self.end)
758
- ```
759
-
760
- - Loading models
761
- ```python
762
- @step
763
- def train(self):
764
- # current.model.load returns the path to the model loaded
765
- checkpoint_path = current.model.load(
766
- self.checkpoint_key,
767
- )
768
- model_path = current.model.load(
769
- self.model,
770
- )
771
- self.next(self.test)
772
- ```
773
-
774
-
775
- Parameters
776
- ----------
777
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
778
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
779
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
780
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
781
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
782
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
783
-
784
- temp_dir_root : str, default: None
785
- The root directory under which `current.model.loaded` will store loaded models
786
- """
787
- ...
788
-
789
- @typing.overload
790
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
791
- """
792
- Specifies environment variables to be set prior to the execution of a step.
793
-
794
-
795
- Parameters
796
- ----------
797
- vars : Dict[str, str], default {}
798
- Dictionary of environment variables to set.
799
- """
800
- ...
801
-
802
- @typing.overload
803
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
804
- ...
805
-
806
- @typing.overload
807
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
808
- ...
809
-
810
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
811
- """
812
- Specifies environment variables to be set prior to the execution of a step.
813
-
814
-
815
- Parameters
816
- ----------
817
- vars : Dict[str, str], default {}
818
- Dictionary of environment variables to set.
819
- """
820
- ...
821
-
822
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
823
- """
824
- Specifies that this step should execute on Kubernetes.
825
-
826
-
827
- Parameters
828
- ----------
829
- cpu : int, default 1
830
- Number of CPUs required for this step. If `@resources` is
831
- also present, the maximum value from all decorators is used.
832
- memory : int, default 4096
833
- Memory size (in MB) required for this step. If
834
- `@resources` is also present, the maximum value from all decorators is
835
- used.
836
- disk : int, default 10240
837
- Disk size (in MB) required for this step. If
838
- `@resources` is also present, the maximum value from all decorators is
839
- used.
840
- image : str, optional, default None
841
- Docker image to use when launching on Kubernetes. If not specified, and
842
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
843
- not, a default Docker image mapping to the current version of Python is used.
844
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
845
- If given, the imagePullPolicy to be applied to the Docker image of the step.
846
- image_pull_secrets: List[str], default []
847
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
848
- Kubernetes image pull secrets to use when pulling container images
849
- in Kubernetes.
850
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
851
- Kubernetes service account to use when launching pod in Kubernetes.
852
- secrets : List[str], optional, default None
853
- Kubernetes secrets to use when launching pod in Kubernetes. These
854
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
855
- in Metaflow configuration.
856
- node_selector: Union[Dict[str,str], str], optional, default None
857
- Kubernetes node selector(s) to apply to the pod running the task.
858
- Can be passed in as a comma separated string of values e.g.
859
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
860
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
861
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
862
- Kubernetes namespace to use when launching pod in Kubernetes.
863
- gpu : int, optional, default None
864
- Number of GPUs required for this step. A value of zero implies that
865
- the scheduled node should not have GPUs.
866
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
867
- The vendor of the GPUs to be used for this step.
868
- tolerations : List[str], default []
869
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
870
- Kubernetes tolerations to use when launching pod in Kubernetes.
871
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
872
- Kubernetes labels to use when launching pod in Kubernetes.
873
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
874
- Kubernetes annotations to use when launching pod in Kubernetes.
875
- use_tmpfs : bool, default False
876
- This enables an explicit tmpfs mount for this step.
877
- tmpfs_tempdir : bool, default True
878
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
879
- tmpfs_size : int, optional, default: None
880
- The value for the size (in MiB) of the tmpfs mount for this step.
881
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
882
- memory allocated for this step.
883
- tmpfs_path : str, optional, default /metaflow_temp
884
- Path to tmpfs mount for this step.
885
- persistent_volume_claims : Dict[str, str], optional, default None
886
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
887
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
888
- shared_memory: int, optional
889
- Shared memory size (in MiB) required for this step
890
- port: int, optional
891
- Port number to specify in the Kubernetes job object
892
- compute_pool : str, optional, default None
893
- Compute pool to be used for for this step.
894
- If not specified, any accessible compute pool within the perimeter is used.
895
- hostname_resolution_timeout: int, default 10 * 60
896
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
897
- Only applicable when @parallel is used.
898
- qos: str, default: Burstable
899
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
900
-
901
- security_context: Dict[str, Any], optional, default None
902
- Container security context. Applies to the task container. Allows the following keys:
903
- - privileged: bool, optional, default None
904
- - allow_privilege_escalation: bool, optional, default None
905
- - run_as_user: int, optional, default None
906
- - run_as_group: int, optional, default None
907
- - run_as_non_root: bool, optional, default None
908
- """
909
- ...
910
-
911
- @typing.overload
912
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
913
- """
914
- Enables checkpointing for a step.
915
-
916
- > Examples
917
-
918
- - Saving Checkpoints
919
-
920
- ```python
921
- @checkpoint
698
+ @checkpoint
922
699
  @step
923
700
  def train(self):
924
701
  model = create_model(self.parameters, checkpoint_path = None)
@@ -1055,125 +832,97 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
1055
832
  """
1056
833
  ...
1057
834
 
1058
- @typing.overload
1059
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
835
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1060
836
  """
1061
- Specifies the number of times the task corresponding
1062
- to a step needs to be retried.
837
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
1063
838
 
1064
- This decorator is useful for handling transient errors, such as networking issues.
1065
- If your task contains operations that can't be retried safely, e.g. database updates,
1066
- it is advisable to annotate it with `@retry(times=0)`.
839
+ User code call
840
+ --------------
841
+ @ollama(
842
+ models=[...],
843
+ ...
844
+ )
1067
845
 
1068
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1069
- decorator will execute a no-op task after all retries have been exhausted,
1070
- ensuring that the flow execution can continue.
846
+ Valid backend options
847
+ ---------------------
848
+ - 'local': Run as a separate process on the local task machine.
849
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
850
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
851
+
852
+ Valid model options
853
+ -------------------
854
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
1071
855
 
1072
856
 
1073
857
  Parameters
1074
858
  ----------
1075
- times : int, default 3
1076
- Number of times to retry this task.
1077
- minutes_between_retries : int, default 2
1078
- Number of minutes between retries.
859
+ models: list[str]
860
+ List of Ollama containers running models in sidecars.
861
+ backend: str
862
+ Determines where and how to run the Ollama process.
863
+ force_pull: bool
864
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
865
+ cache_update_policy: str
866
+ Cache update policy: "auto", "force", or "never".
867
+ force_cache_update: bool
868
+ Simple override for "force" cache update policy.
869
+ debug: bool
870
+ Whether to turn on verbose debugging logs.
871
+ circuit_breaker_config: dict
872
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
873
+ timeout_config: dict
874
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
1079
875
  """
1080
876
  ...
1081
877
 
1082
878
  @typing.overload
1083
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1084
- ...
1085
-
1086
- @typing.overload
1087
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1088
- ...
1089
-
1090
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
879
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1091
880
  """
1092
- Specifies the number of times the task corresponding
1093
- to a step needs to be retried.
1094
-
1095
- This decorator is useful for handling transient errors, such as networking issues.
1096
- If your task contains operations that can't be retried safely, e.g. database updates,
1097
- it is advisable to annotate it with `@retry(times=0)`.
1098
-
1099
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1100
- decorator will execute a no-op task after all retries have been exhausted,
1101
- ensuring that the flow execution can continue.
1102
-
881
+ Specifies the PyPI packages for the step.
1103
882
 
1104
- Parameters
1105
- ----------
1106
- times : int, default 3
1107
- Number of times to retry this task.
1108
- minutes_between_retries : int, default 2
1109
- Number of minutes between retries.
1110
- """
1111
- ...
1112
-
1113
- @typing.overload
1114
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1115
- """
1116
- Specifies secrets to be retrieved and injected as environment variables prior to
1117
- the execution of a step.
883
+ Information in this decorator will augment any
884
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
885
+ you can use `@pypi_base` to set packages required by all
886
+ steps and use `@pypi` to specify step-specific overrides.
1118
887
 
1119
888
 
1120
889
  Parameters
1121
890
  ----------
1122
- sources : List[Union[str, Dict[str, Any]]], default: []
1123
- List of secret specs, defining how the secrets are to be retrieved
891
+ packages : Dict[str, str], default: {}
892
+ Packages to use for this step. The key is the name of the package
893
+ and the value is the version to use.
894
+ python : str, optional, default: None
895
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
896
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1124
897
  """
1125
898
  ...
1126
899
 
1127
900
  @typing.overload
1128
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
901
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1129
902
  ...
1130
903
 
1131
904
  @typing.overload
1132
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
905
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1133
906
  ...
1134
907
 
1135
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
908
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1136
909
  """
1137
- Specifies secrets to be retrieved and injected as environment variables prior to
1138
- the execution of a step.
1139
-
910
+ Specifies the PyPI packages for the step.
1140
911
 
1141
- Parameters
1142
- ----------
1143
- sources : List[Union[str, Dict[str, Any]]], default: []
1144
- List of secret specs, defining how the secrets are to be retrieved
1145
- """
1146
- ...
1147
-
1148
- @typing.overload
1149
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1150
- """
1151
- Internal decorator to support Fast bakery
1152
- """
1153
- ...
1154
-
1155
- @typing.overload
1156
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1157
- ...
1158
-
1159
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1160
- """
1161
- Internal decorator to support Fast bakery
1162
- """
1163
- ...
1164
-
1165
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1166
- """
1167
- Specifies that this step is used to deploy an instance of the app.
1168
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
912
+ Information in this decorator will augment any
913
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
914
+ you can use `@pypi_base` to set packages required by all
915
+ steps and use `@pypi` to specify step-specific overrides.
1169
916
 
1170
917
 
1171
918
  Parameters
1172
919
  ----------
1173
- app_port : int
1174
- Number of GPUs to use.
1175
- app_name : str
1176
- Name of the app to deploy.
920
+ packages : Dict[str, str], default: {}
921
+ Packages to use for this step. The key is the name of the package
922
+ and the value is the version to use.
923
+ python : str, optional, default: None
924
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
925
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1177
926
  """
1178
927
  ...
1179
928
 
@@ -1197,92 +946,307 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
1197
946
  ...
1198
947
 
1199
948
  @typing.overload
1200
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
949
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1201
950
  """
1202
- Creates a human-readable report, a Metaflow Card, after this step completes.
951
+ Enables loading / saving of models within a step.
1203
952
 
1204
- Note that you may add multiple `@card` decorators in a step with different parameters.
953
+ > Examples
954
+ - Saving Models
955
+ ```python
956
+ @model
957
+ @step
958
+ def train(self):
959
+ # current.model.save returns a dictionary reference to the model saved
960
+ self.my_model = current.model.save(
961
+ path_to_my_model,
962
+ label="my_model",
963
+ metadata={
964
+ "epochs": 10,
965
+ "batch-size": 32,
966
+ "learning-rate": 0.001,
967
+ }
968
+ )
969
+ self.next(self.test)
970
+
971
+ @model(load="my_model")
972
+ @step
973
+ def test(self):
974
+ # `current.model.loaded` returns a dictionary of the loaded models
975
+ # where the key is the name of the artifact and the value is the path to the model
976
+ print(os.listdir(current.model.loaded["my_model"]))
977
+ self.next(self.end)
978
+ ```
979
+
980
+ - Loading models
981
+ ```python
982
+ @step
983
+ def train(self):
984
+ # current.model.load returns the path to the model loaded
985
+ checkpoint_path = current.model.load(
986
+ self.checkpoint_key,
987
+ )
988
+ model_path = current.model.load(
989
+ self.model,
990
+ )
991
+ self.next(self.test)
992
+ ```
1205
993
 
1206
994
 
1207
995
  Parameters
1208
996
  ----------
1209
- type : str, default 'default'
1210
- Card type.
1211
- id : str, optional, default None
1212
- If multiple cards are present, use this id to identify this card.
1213
- options : Dict[str, Any], default {}
1214
- Options passed to the card. The contents depend on the card type.
1215
- timeout : int, default 45
1216
- Interrupt reporting if it takes more than this many seconds.
997
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
998
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
999
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1000
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1001
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1002
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1003
+
1004
+ temp_dir_root : str, default: None
1005
+ The root directory under which `current.model.loaded` will store loaded models
1217
1006
  """
1218
1007
  ...
1219
1008
 
1220
1009
  @typing.overload
1221
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1010
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1222
1011
  ...
1223
1012
 
1224
1013
  @typing.overload
1225
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1014
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1226
1015
  ...
1227
1016
 
1228
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1017
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1229
1018
  """
1230
- Creates a human-readable report, a Metaflow Card, after this step completes.
1019
+ Enables loading / saving of models within a step.
1231
1020
 
1232
- Note that you may add multiple `@card` decorators in a step with different parameters.
1021
+ > Examples
1022
+ - Saving Models
1023
+ ```python
1024
+ @model
1025
+ @step
1026
+ def train(self):
1027
+ # current.model.save returns a dictionary reference to the model saved
1028
+ self.my_model = current.model.save(
1029
+ path_to_my_model,
1030
+ label="my_model",
1031
+ metadata={
1032
+ "epochs": 10,
1033
+ "batch-size": 32,
1034
+ "learning-rate": 0.001,
1035
+ }
1036
+ )
1037
+ self.next(self.test)
1038
+
1039
+ @model(load="my_model")
1040
+ @step
1041
+ def test(self):
1042
+ # `current.model.loaded` returns a dictionary of the loaded models
1043
+ # where the key is the name of the artifact and the value is the path to the model
1044
+ print(os.listdir(current.model.loaded["my_model"]))
1045
+ self.next(self.end)
1046
+ ```
1047
+
1048
+ - Loading models
1049
+ ```python
1050
+ @step
1051
+ def train(self):
1052
+ # current.model.load returns the path to the model loaded
1053
+ checkpoint_path = current.model.load(
1054
+ self.checkpoint_key,
1055
+ )
1056
+ model_path = current.model.load(
1057
+ self.model,
1058
+ )
1059
+ self.next(self.test)
1060
+ ```
1233
1061
 
1234
1062
 
1235
1063
  Parameters
1236
1064
  ----------
1237
- type : str, default 'default'
1238
- Card type.
1239
- id : str, optional, default None
1240
- If multiple cards are present, use this id to identify this card.
1241
- options : Dict[str, Any], default {}
1242
- Options passed to the card. The contents depend on the card type.
1243
- timeout : int, default 45
1244
- Interrupt reporting if it takes more than this many seconds.
1065
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1066
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1067
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1068
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1069
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1070
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1071
+
1072
+ temp_dir_root : str, default: None
1073
+ The root directory under which `current.model.loaded` will store loaded models
1245
1074
  """
1246
1075
  ...
1247
1076
 
1248
- @typing.overload
1249
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1077
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1250
1078
  """
1251
- Specifies the PyPI packages for all steps of the flow.
1079
+ Specifies that this step should execute on DGX cloud.
1252
1080
 
1253
- Use `@pypi_base` to set common packages required by all
1254
- steps and use `@pypi` to specify step-specific overrides.
1255
1081
 
1256
1082
  Parameters
1257
1083
  ----------
1258
- packages : Dict[str, str], default: {}
1259
- Packages to use for this flow. The key is the name of the package
1260
- and the value is the version to use.
1261
- python : str, optional, default: None
1262
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1263
- that the version used will correspond to the version of the Python interpreter used to start the run.
1264
- """
1265
- ...
1266
-
1084
+ gpu : int
1085
+ Number of GPUs to use.
1086
+ gpu_type : str
1087
+ Type of Nvidia GPU to use.
1088
+ """
1089
+ ...
1090
+
1267
1091
  @typing.overload
1268
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1092
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1093
+ """
1094
+ Specifies secrets to be retrieved and injected as environment variables prior to
1095
+ the execution of a step.
1096
+
1097
+
1098
+ Parameters
1099
+ ----------
1100
+ sources : List[Union[str, Dict[str, Any]]], default: []
1101
+ List of secret specs, defining how the secrets are to be retrieved
1102
+ """
1269
1103
  ...
1270
1104
 
1271
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1105
+ @typing.overload
1106
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1107
+ ...
1108
+
1109
+ @typing.overload
1110
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1111
+ ...
1112
+
1113
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1272
1114
  """
1273
- Specifies the PyPI packages for all steps of the flow.
1115
+ Specifies secrets to be retrieved and injected as environment variables prior to
1116
+ the execution of a step.
1274
1117
 
1275
- Use `@pypi_base` to set common packages required by all
1276
- steps and use `@pypi` to specify step-specific overrides.
1277
1118
 
1278
1119
  Parameters
1279
1120
  ----------
1280
- packages : Dict[str, str], default: {}
1281
- Packages to use for this flow. The key is the name of the package
1282
- and the value is the version to use.
1283
- python : str, optional, default: None
1284
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1285
- that the version used will correspond to the version of the Python interpreter used to start the run.
1121
+ sources : List[Union[str, Dict[str, Any]]], default: []
1122
+ List of secret specs, defining how the secrets are to be retrieved
1123
+ """
1124
+ ...
1125
+
1126
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1127
+ """
1128
+ Specifies that this step should execute on DGX cloud.
1129
+
1130
+
1131
+ Parameters
1132
+ ----------
1133
+ gpu : int
1134
+ Number of GPUs to use.
1135
+ gpu_type : str
1136
+ Type of Nvidia GPU to use.
1137
+ queue_timeout : int
1138
+ Time to keep the job in NVCF's queue.
1139
+ """
1140
+ ...
1141
+
1142
+ @typing.overload
1143
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1144
+ """
1145
+ Specifies environment variables to be set prior to the execution of a step.
1146
+
1147
+
1148
+ Parameters
1149
+ ----------
1150
+ vars : Dict[str, str], default {}
1151
+ Dictionary of environment variables to set.
1152
+ """
1153
+ ...
1154
+
1155
+ @typing.overload
1156
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1157
+ ...
1158
+
1159
+ @typing.overload
1160
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1161
+ ...
1162
+
1163
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1164
+ """
1165
+ Specifies environment variables to be set prior to the execution of a step.
1166
+
1167
+
1168
+ Parameters
1169
+ ----------
1170
+ vars : Dict[str, str], default {}
1171
+ Dictionary of environment variables to set.
1172
+ """
1173
+ ...
1174
+
1175
+ @typing.overload
1176
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1177
+ """
1178
+ Decorator prototype for all step decorators. This function gets specialized
1179
+ and imported for all decorators types by _import_plugin_decorators().
1180
+ """
1181
+ ...
1182
+
1183
+ @typing.overload
1184
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1185
+ ...
1186
+
1187
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1188
+ """
1189
+ Decorator prototype for all step decorators. This function gets specialized
1190
+ and imported for all decorators types by _import_plugin_decorators().
1191
+ """
1192
+ ...
1193
+
1194
+ @typing.overload
1195
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1196
+ """
1197
+ Specifies a timeout for your step.
1198
+
1199
+ This decorator is useful if this step may hang indefinitely.
1200
+
1201
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1202
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1203
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1204
+
1205
+ Note that all the values specified in parameters are added together so if you specify
1206
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1207
+
1208
+
1209
+ Parameters
1210
+ ----------
1211
+ seconds : int, default 0
1212
+ Number of seconds to wait prior to timing out.
1213
+ minutes : int, default 0
1214
+ Number of minutes to wait prior to timing out.
1215
+ hours : int, default 0
1216
+ Number of hours to wait prior to timing out.
1217
+ """
1218
+ ...
1219
+
1220
+ @typing.overload
1221
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1222
+ ...
1223
+
1224
+ @typing.overload
1225
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1226
+ ...
1227
+
1228
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1229
+ """
1230
+ Specifies a timeout for your step.
1231
+
1232
+ This decorator is useful if this step may hang indefinitely.
1233
+
1234
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1235
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1236
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1237
+
1238
+ Note that all the values specified in parameters are added together so if you specify
1239
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1240
+
1241
+
1242
+ Parameters
1243
+ ----------
1244
+ seconds : int, default 0
1245
+ Number of seconds to wait prior to timing out.
1246
+ minutes : int, default 0
1247
+ Number of minutes to wait prior to timing out.
1248
+ hours : int, default 0
1249
+ Number of hours to wait prior to timing out.
1286
1250
  """
1287
1251
  ...
1288
1252
 
@@ -1400,283 +1364,87 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1400
1364
  """
1401
1365
  ...
1402
1366
 
1403
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1404
- """
1405
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1406
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1407
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1408
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1409
- starts only after all sensors finish.
1410
-
1411
-
1412
- Parameters
1413
- ----------
1414
- timeout : int
1415
- Time, in seconds before the task times out and fails. (Default: 3600)
1416
- poke_interval : int
1417
- Time in seconds that the job should wait in between each try. (Default: 60)
1418
- mode : str
1419
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1420
- exponential_backoff : bool
1421
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1422
- pool : str
1423
- the slot pool this task should run in,
1424
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1425
- soft_fail : bool
1426
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1427
- name : str
1428
- Name of the sensor on Airflow
1429
- description : str
1430
- Description of sensor in the Airflow UI
1431
- bucket_key : Union[str, List[str]]
1432
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1433
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1434
- bucket_name : str
1435
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1436
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1437
- wildcard_match : bool
1438
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1439
- aws_conn_id : str
1440
- a reference to the s3 connection on Airflow. (Default: None)
1441
- verify : bool
1442
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1443
- """
1444
- ...
1445
-
1446
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1447
- """
1448
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1449
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1450
-
1451
-
1452
- Parameters
1453
- ----------
1454
- timeout : int
1455
- Time, in seconds before the task times out and fails. (Default: 3600)
1456
- poke_interval : int
1457
- Time in seconds that the job should wait in between each try. (Default: 60)
1458
- mode : str
1459
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1460
- exponential_backoff : bool
1461
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1462
- pool : str
1463
- the slot pool this task should run in,
1464
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1465
- soft_fail : bool
1466
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1467
- name : str
1468
- Name of the sensor on Airflow
1469
- description : str
1470
- Description of sensor in the Airflow UI
1471
- external_dag_id : str
1472
- The dag_id that contains the task you want to wait for.
1473
- external_task_ids : List[str]
1474
- The list of task_ids that you want to wait for.
1475
- If None (default value) the sensor waits for the DAG. (Default: None)
1476
- allowed_states : List[str]
1477
- Iterable of allowed states, (Default: ['success'])
1478
- failed_states : List[str]
1479
- Iterable of failed or dis-allowed states. (Default: None)
1480
- execution_delta : datetime.timedelta
1481
- time difference with the previous execution to look at,
1482
- the default is the same logical date as the current task or DAG. (Default: None)
1483
- check_existence: bool
1484
- Set to True to check if the external task exists or check if
1485
- the DAG to wait for exists. (Default: True)
1486
- """
1487
- ...
1488
-
1489
1367
  @typing.overload
1490
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1368
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1491
1369
  """
1492
- Specifies the event(s) that this flow depends on.
1493
-
1494
- ```
1495
- @trigger(event='foo')
1496
- ```
1497
- or
1498
- ```
1499
- @trigger(events=['foo', 'bar'])
1500
- ```
1501
-
1502
- Additionally, you can specify the parameter mappings
1503
- to map event payload to Metaflow parameters for the flow.
1504
- ```
1505
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1506
- ```
1507
- or
1508
- ```
1509
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1510
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1511
- ```
1512
-
1513
- 'parameters' can also be a list of strings and tuples like so:
1514
- ```
1515
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1516
- ```
1517
- This is equivalent to:
1518
- ```
1519
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1520
- ```
1370
+ Specifies the PyPI packages for all steps of the flow.
1521
1371
 
1372
+ Use `@pypi_base` to set common packages required by all
1373
+ steps and use `@pypi` to specify step-specific overrides.
1522
1374
 
1523
1375
  Parameters
1524
1376
  ----------
1525
- event : Union[str, Dict[str, Any]], optional, default None
1526
- Event dependency for this flow.
1527
- events : List[Union[str, Dict[str, Any]]], default []
1528
- Events dependency for this flow.
1529
- options : Dict[str, Any], default {}
1530
- Backend-specific configuration for tuning eventing behavior.
1377
+ packages : Dict[str, str], default: {}
1378
+ Packages to use for this flow. The key is the name of the package
1379
+ and the value is the version to use.
1380
+ python : str, optional, default: None
1381
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1382
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1531
1383
  """
1532
1384
  ...
1533
1385
 
1534
1386
  @typing.overload
1535
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1536
- ...
1537
-
1538
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1539
- """
1540
- Specifies the event(s) that this flow depends on.
1541
-
1542
- ```
1543
- @trigger(event='foo')
1544
- ```
1545
- or
1546
- ```
1547
- @trigger(events=['foo', 'bar'])
1548
- ```
1549
-
1550
- Additionally, you can specify the parameter mappings
1551
- to map event payload to Metaflow parameters for the flow.
1552
- ```
1553
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1554
- ```
1555
- or
1556
- ```
1557
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1558
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1559
- ```
1560
-
1561
- 'parameters' can also be a list of strings and tuples like so:
1562
- ```
1563
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1564
- ```
1565
- This is equivalent to:
1566
- ```
1567
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1568
- ```
1569
-
1570
-
1571
- Parameters
1572
- ----------
1573
- event : Union[str, Dict[str, Any]], optional, default None
1574
- Event dependency for this flow.
1575
- events : List[Union[str, Dict[str, Any]]], default []
1576
- Events dependency for this flow.
1577
- options : Dict[str, Any], default {}
1578
- Backend-specific configuration for tuning eventing behavior.
1579
- """
1387
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1580
1388
  ...
1581
1389
 
1582
- @typing.overload
1583
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1390
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1584
1391
  """
1585
- Specifies the flow(s) that this flow depends on.
1586
-
1587
- ```
1588
- @trigger_on_finish(flow='FooFlow')
1589
- ```
1590
- or
1591
- ```
1592
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1593
- ```
1594
- This decorator respects the @project decorator and triggers the flow
1595
- when upstream runs within the same namespace complete successfully
1596
-
1597
- Additionally, you can specify project aware upstream flow dependencies
1598
- by specifying the fully qualified project_flow_name.
1599
- ```
1600
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1601
- ```
1602
- or
1603
- ```
1604
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1605
- ```
1606
-
1607
- You can also specify just the project or project branch (other values will be
1608
- inferred from the current project or project branch):
1609
- ```
1610
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1611
- ```
1612
-
1613
- Note that `branch` is typically one of:
1614
- - `prod`
1615
- - `user.bob`
1616
- - `test.my_experiment`
1617
- - `prod.staging`
1392
+ Specifies the PyPI packages for all steps of the flow.
1618
1393
 
1394
+ Use `@pypi_base` to set common packages required by all
1395
+ steps and use `@pypi` to specify step-specific overrides.
1619
1396
 
1620
1397
  Parameters
1621
1398
  ----------
1622
- flow : Union[str, Dict[str, str]], optional, default None
1623
- Upstream flow dependency for this flow.
1624
- flows : List[Union[str, Dict[str, str]]], default []
1625
- Upstream flow dependencies for this flow.
1626
- options : Dict[str, Any], default {}
1627
- Backend-specific configuration for tuning eventing behavior.
1399
+ packages : Dict[str, str], default: {}
1400
+ Packages to use for this flow. The key is the name of the package
1401
+ and the value is the version to use.
1402
+ python : str, optional, default: None
1403
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1404
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1628
1405
  """
1629
1406
  ...
1630
1407
 
1631
- @typing.overload
1632
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1633
- ...
1634
-
1635
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1408
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1636
1409
  """
1637
- Specifies the flow(s) that this flow depends on.
1638
-
1639
- ```
1640
- @trigger_on_finish(flow='FooFlow')
1641
- ```
1642
- or
1643
- ```
1644
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1645
- ```
1646
- This decorator respects the @project decorator and triggers the flow
1647
- when upstream runs within the same namespace complete successfully
1648
-
1649
- Additionally, you can specify project aware upstream flow dependencies
1650
- by specifying the fully qualified project_flow_name.
1651
- ```
1652
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1653
- ```
1654
- or
1655
- ```
1656
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1657
- ```
1658
-
1659
- You can also specify just the project or project branch (other values will be
1660
- inferred from the current project or project branch):
1661
- ```
1662
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1663
- ```
1664
-
1665
- Note that `branch` is typically one of:
1666
- - `prod`
1667
- - `user.bob`
1668
- - `test.my_experiment`
1669
- - `prod.staging`
1410
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1411
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1412
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1413
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1414
+ starts only after all sensors finish.
1670
1415
 
1671
1416
 
1672
1417
  Parameters
1673
1418
  ----------
1674
- flow : Union[str, Dict[str, str]], optional, default None
1675
- Upstream flow dependency for this flow.
1676
- flows : List[Union[str, Dict[str, str]]], default []
1677
- Upstream flow dependencies for this flow.
1678
- options : Dict[str, Any], default {}
1679
- Backend-specific configuration for tuning eventing behavior.
1419
+ timeout : int
1420
+ Time, in seconds before the task times out and fails. (Default: 3600)
1421
+ poke_interval : int
1422
+ Time in seconds that the job should wait in between each try. (Default: 60)
1423
+ mode : str
1424
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1425
+ exponential_backoff : bool
1426
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1427
+ pool : str
1428
+ the slot pool this task should run in,
1429
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1430
+ soft_fail : bool
1431
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1432
+ name : str
1433
+ Name of the sensor on Airflow
1434
+ description : str
1435
+ Description of sensor in the Airflow UI
1436
+ bucket_key : Union[str, List[str]]
1437
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1438
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1439
+ bucket_name : str
1440
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1441
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1442
+ wildcard_match : bool
1443
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1444
+ aws_conn_id : str
1445
+ a reference to the s3 connection on Airflow. (Default: None)
1446
+ verify : bool
1447
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1680
1448
  """
1681
1449
  ...
1682
1450
 
@@ -1731,6 +1499,142 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1731
1499
  """
1732
1500
  ...
1733
1501
 
1502
+ @typing.overload
1503
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1504
+ """
1505
+ Specifies the event(s) that this flow depends on.
1506
+
1507
+ ```
1508
+ @trigger(event='foo')
1509
+ ```
1510
+ or
1511
+ ```
1512
+ @trigger(events=['foo', 'bar'])
1513
+ ```
1514
+
1515
+ Additionally, you can specify the parameter mappings
1516
+ to map event payload to Metaflow parameters for the flow.
1517
+ ```
1518
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1519
+ ```
1520
+ or
1521
+ ```
1522
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1523
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1524
+ ```
1525
+
1526
+ 'parameters' can also be a list of strings and tuples like so:
1527
+ ```
1528
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1529
+ ```
1530
+ This is equivalent to:
1531
+ ```
1532
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1533
+ ```
1534
+
1535
+
1536
+ Parameters
1537
+ ----------
1538
+ event : Union[str, Dict[str, Any]], optional, default None
1539
+ Event dependency for this flow.
1540
+ events : List[Union[str, Dict[str, Any]]], default []
1541
+ Events dependency for this flow.
1542
+ options : Dict[str, Any], default {}
1543
+ Backend-specific configuration for tuning eventing behavior.
1544
+ """
1545
+ ...
1546
+
1547
+ @typing.overload
1548
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1549
+ ...
1550
+
1551
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1552
+ """
1553
+ Specifies the event(s) that this flow depends on.
1554
+
1555
+ ```
1556
+ @trigger(event='foo')
1557
+ ```
1558
+ or
1559
+ ```
1560
+ @trigger(events=['foo', 'bar'])
1561
+ ```
1562
+
1563
+ Additionally, you can specify the parameter mappings
1564
+ to map event payload to Metaflow parameters for the flow.
1565
+ ```
1566
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1567
+ ```
1568
+ or
1569
+ ```
1570
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1571
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1572
+ ```
1573
+
1574
+ 'parameters' can also be a list of strings and tuples like so:
1575
+ ```
1576
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1577
+ ```
1578
+ This is equivalent to:
1579
+ ```
1580
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1581
+ ```
1582
+
1583
+
1584
+ Parameters
1585
+ ----------
1586
+ event : Union[str, Dict[str, Any]], optional, default None
1587
+ Event dependency for this flow.
1588
+ events : List[Union[str, Dict[str, Any]]], default []
1589
+ Events dependency for this flow.
1590
+ options : Dict[str, Any], default {}
1591
+ Backend-specific configuration for tuning eventing behavior.
1592
+ """
1593
+ ...
1594
+
1595
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1596
+ """
1597
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1598
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1599
+
1600
+
1601
+ Parameters
1602
+ ----------
1603
+ timeout : int
1604
+ Time, in seconds before the task times out and fails. (Default: 3600)
1605
+ poke_interval : int
1606
+ Time in seconds that the job should wait in between each try. (Default: 60)
1607
+ mode : str
1608
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1609
+ exponential_backoff : bool
1610
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1611
+ pool : str
1612
+ the slot pool this task should run in,
1613
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1614
+ soft_fail : bool
1615
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1616
+ name : str
1617
+ Name of the sensor on Airflow
1618
+ description : str
1619
+ Description of sensor in the Airflow UI
1620
+ external_dag_id : str
1621
+ The dag_id that contains the task you want to wait for.
1622
+ external_task_ids : List[str]
1623
+ The list of task_ids that you want to wait for.
1624
+ If None (default value) the sensor waits for the DAG. (Default: None)
1625
+ allowed_states : List[str]
1626
+ Iterable of allowed states, (Default: ['success'])
1627
+ failed_states : List[str]
1628
+ Iterable of failed or dis-allowed states. (Default: None)
1629
+ execution_delta : datetime.timedelta
1630
+ time difference with the previous execution to look at,
1631
+ the default is the same logical date as the current task or DAG. (Default: None)
1632
+ check_existence: bool
1633
+ Set to True to check if the external task exists or check if
1634
+ the DAG to wait for exists. (Default: True)
1635
+ """
1636
+ ...
1637
+
1734
1638
  @typing.overload
1735
1639
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1736
1640
  """
@@ -1817,5 +1721,106 @@ def project(*, name: str, branch: typing.Optional[str] = None, production: bool
1817
1721
  """
1818
1722
  ...
1819
1723
 
1724
+ @typing.overload
1725
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1726
+ """
1727
+ Specifies the flow(s) that this flow depends on.
1728
+
1729
+ ```
1730
+ @trigger_on_finish(flow='FooFlow')
1731
+ ```
1732
+ or
1733
+ ```
1734
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1735
+ ```
1736
+ This decorator respects the @project decorator and triggers the flow
1737
+ when upstream runs within the same namespace complete successfully
1738
+
1739
+ Additionally, you can specify project aware upstream flow dependencies
1740
+ by specifying the fully qualified project_flow_name.
1741
+ ```
1742
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1743
+ ```
1744
+ or
1745
+ ```
1746
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1747
+ ```
1748
+
1749
+ You can also specify just the project or project branch (other values will be
1750
+ inferred from the current project or project branch):
1751
+ ```
1752
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1753
+ ```
1754
+
1755
+ Note that `branch` is typically one of:
1756
+ - `prod`
1757
+ - `user.bob`
1758
+ - `test.my_experiment`
1759
+ - `prod.staging`
1760
+
1761
+
1762
+ Parameters
1763
+ ----------
1764
+ flow : Union[str, Dict[str, str]], optional, default None
1765
+ Upstream flow dependency for this flow.
1766
+ flows : List[Union[str, Dict[str, str]]], default []
1767
+ Upstream flow dependencies for this flow.
1768
+ options : Dict[str, Any], default {}
1769
+ Backend-specific configuration for tuning eventing behavior.
1770
+ """
1771
+ ...
1772
+
1773
+ @typing.overload
1774
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1775
+ ...
1776
+
1777
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1778
+ """
1779
+ Specifies the flow(s) that this flow depends on.
1780
+
1781
+ ```
1782
+ @trigger_on_finish(flow='FooFlow')
1783
+ ```
1784
+ or
1785
+ ```
1786
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1787
+ ```
1788
+ This decorator respects the @project decorator and triggers the flow
1789
+ when upstream runs within the same namespace complete successfully
1790
+
1791
+ Additionally, you can specify project aware upstream flow dependencies
1792
+ by specifying the fully qualified project_flow_name.
1793
+ ```
1794
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1795
+ ```
1796
+ or
1797
+ ```
1798
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1799
+ ```
1800
+
1801
+ You can also specify just the project or project branch (other values will be
1802
+ inferred from the current project or project branch):
1803
+ ```
1804
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1805
+ ```
1806
+
1807
+ Note that `branch` is typically one of:
1808
+ - `prod`
1809
+ - `user.bob`
1810
+ - `test.my_experiment`
1811
+ - `prod.staging`
1812
+
1813
+
1814
+ Parameters
1815
+ ----------
1816
+ flow : Union[str, Dict[str, str]], optional, default None
1817
+ Upstream flow dependency for this flow.
1818
+ flows : List[Union[str, Dict[str, str]]], default []
1819
+ Upstream flow dependencies for this flow.
1820
+ options : Dict[str, Any], default {}
1821
+ Backend-specific configuration for tuning eventing behavior.
1822
+ """
1823
+ ...
1824
+
1820
1825
  pkg_name: str
1821
1826