ob-metaflow-stubs 6.0.8.3__py2.py3-none-any.whl → 6.0.9.0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ob-metaflow-stubs might be problematic. Click here for more details.

Files changed (262) hide show
  1. metaflow-stubs/__init__.pyi +965 -965
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +13 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +2 -2
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +47 -47
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_state_machine.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/__init__.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/_vendor/spinner/spinners.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_cli.pyi +3 -3
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/app_config.pyi +3 -3
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/capsule.pyi +3 -3
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/click_importer.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/__init__.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/code_package/code_packager.pyi +3 -3
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/cli_generator.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/config_utils.pyi +3 -3
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/schema_export.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/typed_configs.pyi +3 -3
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/config/unified_config.pyi +2 -2
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/dependencies.pyi +3 -3
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/deployer.pyi +4 -4
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/experimental/__init__.pyi +2 -2
  87. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/perimeters.pyi +2 -2
  88. metaflow-stubs/mf_extensions/outerbounds/plugins/apps/core/utils.pyi +3 -3
  89. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  90. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +3 -3
  91. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  92. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +3 -3
  93. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  94. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  95. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +4 -4
  96. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +4 -4
  97. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +2 -2
  98. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +2 -2
  99. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
  100. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +2 -2
  101. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  102. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  103. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  104. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  105. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  106. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  107. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  108. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  109. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  110. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  111. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  112. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +3 -3
  113. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  114. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  115. metaflow-stubs/mf_extensions/outerbounds/toplevel/s3_proxy.pyi +2 -2
  116. metaflow-stubs/multicore_utils.pyi +2 -2
  117. metaflow-stubs/ob_internal.pyi +2 -2
  118. metaflow-stubs/packaging_sys/__init__.pyi +5 -5
  119. metaflow-stubs/packaging_sys/backend.pyi +2 -2
  120. metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
  121. metaflow-stubs/packaging_sys/tar_backend.pyi +6 -6
  122. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  123. metaflow-stubs/packaging_sys/v1.pyi +2 -2
  124. metaflow-stubs/parameters.pyi +2 -2
  125. metaflow-stubs/plugins/__init__.pyi +13 -13
  126. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  128. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  129. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  130. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  131. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  132. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  133. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  134. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  135. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  136. metaflow-stubs/plugins/argo/argo_workflows.pyi +9 -3
  137. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  138. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  139. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  140. metaflow-stubs/plugins/argo/exit_hooks.pyi +2 -2
  141. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  142. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  143. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  144. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  146. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  147. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  148. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  150. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  152. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  154. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  155. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  156. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  157. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  158. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  159. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  160. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  161. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  162. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  163. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  164. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  165. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  166. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  167. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  168. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  169. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  170. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  171. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  172. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  173. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  174. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  175. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  176. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  177. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  178. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  179. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  180. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  181. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  182. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  183. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  184. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  185. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  186. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  187. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  188. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  189. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  190. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  191. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  192. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  193. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  194. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  195. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  196. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  197. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  198. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  199. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  200. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  201. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  202. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  203. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  204. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  205. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  206. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  207. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  208. metaflow-stubs/plugins/optuna/__init__.pyi +2 -2
  209. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  210. metaflow-stubs/plugins/perimeters.pyi +2 -2
  211. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  212. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  213. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  214. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  215. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  216. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  217. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  218. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  219. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  220. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  221. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  222. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  223. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  224. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  225. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  226. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  227. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  228. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  229. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  230. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  231. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  232. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  233. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  234. metaflow-stubs/profilers/__init__.pyi +2 -2
  235. metaflow-stubs/pylint_wrapper.pyi +2 -2
  236. metaflow-stubs/runner/__init__.pyi +2 -2
  237. metaflow-stubs/runner/deployer.pyi +4 -4
  238. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  239. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  240. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  241. metaflow-stubs/runner/nbrun.pyi +2 -2
  242. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  243. metaflow-stubs/runner/utils.pyi +2 -2
  244. metaflow-stubs/system/__init__.pyi +2 -2
  245. metaflow-stubs/system/system_logger.pyi +2 -2
  246. metaflow-stubs/system/system_monitor.pyi +2 -2
  247. metaflow-stubs/tagging_util.pyi +2 -2
  248. metaflow-stubs/tuple_util.pyi +2 -2
  249. metaflow-stubs/user_configs/__init__.pyi +2 -2
  250. metaflow-stubs/user_configs/config_options.pyi +2 -2
  251. metaflow-stubs/user_configs/config_parameters.pyi +8 -6
  252. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  253. metaflow-stubs/user_decorators/common.pyi +2 -2
  254. metaflow-stubs/user_decorators/mutable_flow.pyi +5 -5
  255. metaflow-stubs/user_decorators/mutable_step.pyi +3 -3
  256. metaflow-stubs/user_decorators/user_flow_decorator.pyi +3 -3
  257. metaflow-stubs/user_decorators/user_step_decorator.pyi +19 -6
  258. {ob_metaflow_stubs-6.0.8.3.dist-info → ob_metaflow_stubs-6.0.9.0.dist-info}/METADATA +1 -1
  259. ob_metaflow_stubs-6.0.9.0.dist-info/RECORD +262 -0
  260. ob_metaflow_stubs-6.0.8.3.dist-info/RECORD +0 -262
  261. {ob_metaflow_stubs-6.0.8.3.dist-info → ob_metaflow_stubs-6.0.9.0.dist-info}/WHEEL +0 -0
  262. {ob_metaflow_stubs-6.0.8.3.dist-info → ob_metaflow_stubs-6.0.9.0.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.17.1.0+obcheckpoint(0.2.4);ob(v1) #
4
- # Generated on 2025-08-25T21:23:22.454608 #
3
+ # MF version: 2.18.0.1+obcheckpoint(0.2.4);ob(v1) #
4
+ # Generated on 2025-08-27T22:09:03.717615 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -39,18 +39,18 @@ from .user_decorators.user_step_decorator import UserStepDecorator as UserStepDe
39
39
  from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
- from . import cards as cards
43
- from . import events as events
44
42
  from . import metaflow_git as metaflow_git
43
+ from . import cards as cards
45
44
  from . import tuple_util as tuple_util
45
+ from . import events as events
46
46
  from . import runner as runner
47
47
  from . import plugins as plugins
48
48
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
49
49
  from . import includefile as includefile
50
50
  from .includefile import IncludeFile as IncludeFile
51
51
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
52
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
53
52
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
53
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
54
54
  from . import client as client
55
55
  from .client.core import namespace as namespace
56
56
  from .client.core import get_namespace as get_namespace
@@ -168,288 +168,238 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
168
168
  ...
169
169
 
170
170
  @typing.overload
171
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
171
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
172
172
  """
173
- Specifies a timeout for your step.
174
-
175
- This decorator is useful if this step may hang indefinitely.
176
-
177
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
178
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
179
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
173
+ Specifies the PyPI packages for the step.
180
174
 
181
- Note that all the values specified in parameters are added together so if you specify
182
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
175
+ Information in this decorator will augment any
176
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
177
+ you can use `@pypi_base` to set packages required by all
178
+ steps and use `@pypi` to specify step-specific overrides.
183
179
 
184
180
 
185
181
  Parameters
186
182
  ----------
187
- seconds : int, default 0
188
- Number of seconds to wait prior to timing out.
189
- minutes : int, default 0
190
- Number of minutes to wait prior to timing out.
191
- hours : int, default 0
192
- Number of hours to wait prior to timing out.
183
+ packages : Dict[str, str], default: {}
184
+ Packages to use for this step. The key is the name of the package
185
+ and the value is the version to use.
186
+ python : str, optional, default: None
187
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
188
+ that the version used will correspond to the version of the Python interpreter used to start the run.
193
189
  """
194
190
  ...
195
191
 
196
192
  @typing.overload
197
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
193
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
198
194
  ...
199
195
 
200
196
  @typing.overload
201
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
197
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
202
198
  ...
203
199
 
204
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
200
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
205
201
  """
206
- Specifies a timeout for your step.
207
-
208
- This decorator is useful if this step may hang indefinitely.
209
-
210
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
211
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
212
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
202
+ Specifies the PyPI packages for the step.
213
203
 
214
- Note that all the values specified in parameters are added together so if you specify
215
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
204
+ Information in this decorator will augment any
205
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
206
+ you can use `@pypi_base` to set packages required by all
207
+ steps and use `@pypi` to specify step-specific overrides.
216
208
 
217
209
 
218
210
  Parameters
219
211
  ----------
220
- seconds : int, default 0
221
- Number of seconds to wait prior to timing out.
222
- minutes : int, default 0
223
- Number of minutes to wait prior to timing out.
224
- hours : int, default 0
225
- Number of hours to wait prior to timing out.
212
+ packages : Dict[str, str], default: {}
213
+ Packages to use for this step. The key is the name of the package
214
+ and the value is the version to use.
215
+ python : str, optional, default: None
216
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
217
+ that the version used will correspond to the version of the Python interpreter used to start the run.
226
218
  """
227
219
  ...
228
220
 
229
- @typing.overload
230
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
221
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
231
222
  """
232
- Specifies secrets to be retrieved and injected as environment variables prior to
233
- the execution of a step.
223
+ Specifies that this step should execute on Kubernetes.
234
224
 
235
225
 
236
226
  Parameters
237
227
  ----------
238
- sources : List[Union[str, Dict[str, Any]]], default: []
239
- List of secret specs, defining how the secrets are to be retrieved
240
- role : str, optional, default: None
241
- Role to use for fetching secrets
228
+ cpu : int, default 1
229
+ Number of CPUs required for this step. If `@resources` is
230
+ also present, the maximum value from all decorators is used.
231
+ memory : int, default 4096
232
+ Memory size (in MB) required for this step. If
233
+ `@resources` is also present, the maximum value from all decorators is
234
+ used.
235
+ disk : int, default 10240
236
+ Disk size (in MB) required for this step. If
237
+ `@resources` is also present, the maximum value from all decorators is
238
+ used.
239
+ image : str, optional, default None
240
+ Docker image to use when launching on Kubernetes. If not specified, and
241
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
242
+ not, a default Docker image mapping to the current version of Python is used.
243
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
244
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
245
+ image_pull_secrets: List[str], default []
246
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
247
+ Kubernetes image pull secrets to use when pulling container images
248
+ in Kubernetes.
249
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
250
+ Kubernetes service account to use when launching pod in Kubernetes.
251
+ secrets : List[str], optional, default None
252
+ Kubernetes secrets to use when launching pod in Kubernetes. These
253
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
254
+ in Metaflow configuration.
255
+ node_selector: Union[Dict[str,str], str], optional, default None
256
+ Kubernetes node selector(s) to apply to the pod running the task.
257
+ Can be passed in as a comma separated string of values e.g.
258
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
259
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
260
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
261
+ Kubernetes namespace to use when launching pod in Kubernetes.
262
+ gpu : int, optional, default None
263
+ Number of GPUs required for this step. A value of zero implies that
264
+ the scheduled node should not have GPUs.
265
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
266
+ The vendor of the GPUs to be used for this step.
267
+ tolerations : List[Dict[str,str]], default []
268
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
269
+ Kubernetes tolerations to use when launching pod in Kubernetes.
270
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
271
+ Kubernetes labels to use when launching pod in Kubernetes.
272
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
273
+ Kubernetes annotations to use when launching pod in Kubernetes.
274
+ use_tmpfs : bool, default False
275
+ This enables an explicit tmpfs mount for this step.
276
+ tmpfs_tempdir : bool, default True
277
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
278
+ tmpfs_size : int, optional, default: None
279
+ The value for the size (in MiB) of the tmpfs mount for this step.
280
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
281
+ memory allocated for this step.
282
+ tmpfs_path : str, optional, default /metaflow_temp
283
+ Path to tmpfs mount for this step.
284
+ persistent_volume_claims : Dict[str, str], optional, default None
285
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
286
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
287
+ shared_memory: int, optional
288
+ Shared memory size (in MiB) required for this step
289
+ port: int, optional
290
+ Port number to specify in the Kubernetes job object
291
+ compute_pool : str, optional, default None
292
+ Compute pool to be used for for this step.
293
+ If not specified, any accessible compute pool within the perimeter is used.
294
+ hostname_resolution_timeout: int, default 10 * 60
295
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
296
+ Only applicable when @parallel is used.
297
+ qos: str, default: Burstable
298
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
299
+
300
+ security_context: Dict[str, Any], optional, default None
301
+ Container security context. Applies to the task container. Allows the following keys:
302
+ - privileged: bool, optional, default None
303
+ - allow_privilege_escalation: bool, optional, default None
304
+ - run_as_user: int, optional, default None
305
+ - run_as_group: int, optional, default None
306
+ - run_as_non_root: bool, optional, default None
242
307
  """
243
308
  ...
244
309
 
245
310
  @typing.overload
246
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
311
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
312
+ """
313
+ Decorator prototype for all step decorators. This function gets specialized
314
+ and imported for all decorators types by _import_plugin_decorators().
315
+ """
247
316
  ...
248
317
 
249
318
  @typing.overload
250
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
319
+ def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
251
320
  ...
252
321
 
253
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
322
+ def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
254
323
  """
255
- Specifies secrets to be retrieved and injected as environment variables prior to
256
- the execution of a step.
257
-
258
-
259
- Parameters
260
- ----------
261
- sources : List[Union[str, Dict[str, Any]]], default: []
262
- List of secret specs, defining how the secrets are to be retrieved
263
- role : str, optional, default: None
264
- Role to use for fetching secrets
324
+ Decorator prototype for all step decorators. This function gets specialized
325
+ and imported for all decorators types by _import_plugin_decorators().
265
326
  """
266
327
  ...
267
328
 
268
- @typing.overload
269
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
329
+ def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
270
330
  """
271
- Creates a human-readable report, a Metaflow Card, after this step completes.
331
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
272
332
 
273
- Note that you may add multiple `@card` decorators in a step with different parameters.
333
+ User code call
334
+ --------------
335
+ @vllm(
336
+ model="...",
337
+ ...
338
+ )
339
+
340
+ Valid backend options
341
+ ---------------------
342
+ - 'local': Run as a separate process on the local task machine.
343
+
344
+ Valid model options
345
+ -------------------
346
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
347
+
348
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
349
+ If you need multiple models, you must create multiple @vllm decorators.
274
350
 
275
351
 
276
352
  Parameters
277
353
  ----------
278
- type : str, default 'default'
279
- Card type.
280
- id : str, optional, default None
281
- If multiple cards are present, use this id to identify this card.
282
- options : Dict[str, Any], default {}
283
- Options passed to the card. The contents depend on the card type.
284
- timeout : int, default 45
285
- Interrupt reporting if it takes more than this many seconds.
354
+ model: str
355
+ HuggingFace model identifier to be served by vLLM.
356
+ backend: str
357
+ Determines where and how to run the vLLM process.
358
+ openai_api_server: bool
359
+ Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
360
+ Default is False (uses native engine).
361
+ Set to True for backward compatibility with existing code.
362
+ debug: bool
363
+ Whether to turn on verbose debugging logs.
364
+ card_refresh_interval: int
365
+ Interval in seconds for refreshing the vLLM status card.
366
+ Only used when openai_api_server=True.
367
+ max_retries: int
368
+ Maximum number of retries checking for vLLM server startup.
369
+ Only used when openai_api_server=True.
370
+ retry_alert_frequency: int
371
+ Frequency of alert logs for vLLM server startup retries.
372
+ Only used when openai_api_server=True.
373
+ engine_args : dict
374
+ Additional keyword arguments to pass to the vLLM engine.
375
+ For example, `tensor_parallel_size=2`.
286
376
  """
287
377
  ...
288
378
 
289
379
  @typing.overload
290
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
380
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
381
+ """
382
+ A simple decorator that demonstrates using CardDecoratorInjector
383
+ to inject a card and render simple markdown content.
384
+ """
291
385
  ...
292
386
 
293
387
  @typing.overload
294
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
388
+ def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
295
389
  ...
296
390
 
297
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
391
+ def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
298
392
  """
299
- Creates a human-readable report, a Metaflow Card, after this step completes.
300
-
301
- Note that you may add multiple `@card` decorators in a step with different parameters.
393
+ A simple decorator that demonstrates using CardDecoratorInjector
394
+ to inject a card and render simple markdown content.
395
+ """
396
+ ...
397
+
398
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
399
+ """
400
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
302
401
 
303
-
304
- Parameters
305
- ----------
306
- type : str, default 'default'
307
- Card type.
308
- id : str, optional, default None
309
- If multiple cards are present, use this id to identify this card.
310
- options : Dict[str, Any], default {}
311
- Options passed to the card. The contents depend on the card type.
312
- timeout : int, default 45
313
- Interrupt reporting if it takes more than this many seconds.
314
- """
315
- ...
316
-
317
- @typing.overload
318
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
319
- """
320
- Specifies the Conda environment for the step.
321
-
322
- Information in this decorator will augment any
323
- attributes set in the `@conda_base` flow-level decorator. Hence,
324
- you can use `@conda_base` to set packages required by all
325
- steps and use `@conda` to specify step-specific overrides.
326
-
327
-
328
- Parameters
329
- ----------
330
- packages : Dict[str, str], default {}
331
- Packages to use for this step. The key is the name of the package
332
- and the value is the version to use.
333
- libraries : Dict[str, str], default {}
334
- Supported for backward compatibility. When used with packages, packages will take precedence.
335
- python : str, optional, default None
336
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
337
- that the version used will correspond to the version of the Python interpreter used to start the run.
338
- disabled : bool, default False
339
- If set to True, disables @conda.
340
- """
341
- ...
342
-
343
- @typing.overload
344
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
345
- ...
346
-
347
- @typing.overload
348
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
349
- ...
350
-
351
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
352
- """
353
- Specifies the Conda environment for the step.
354
-
355
- Information in this decorator will augment any
356
- attributes set in the `@conda_base` flow-level decorator. Hence,
357
- you can use `@conda_base` to set packages required by all
358
- steps and use `@conda` to specify step-specific overrides.
359
-
360
-
361
- Parameters
362
- ----------
363
- packages : Dict[str, str], default {}
364
- Packages to use for this step. The key is the name of the package
365
- and the value is the version to use.
366
- libraries : Dict[str, str], default {}
367
- Supported for backward compatibility. When used with packages, packages will take precedence.
368
- python : str, optional, default None
369
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
370
- that the version used will correspond to the version of the Python interpreter used to start the run.
371
- disabled : bool, default False
372
- If set to True, disables @conda.
373
- """
374
- ...
375
-
376
- @typing.overload
377
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
378
- """
379
- Internal decorator to support Fast bakery
380
- """
381
- ...
382
-
383
- @typing.overload
384
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
385
- ...
386
-
387
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
388
- """
389
- Internal decorator to support Fast bakery
390
- """
391
- ...
392
-
393
- @typing.overload
394
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
395
- """
396
- Specifies the number of times the task corresponding
397
- to a step needs to be retried.
398
-
399
- This decorator is useful for handling transient errors, such as networking issues.
400
- If your task contains operations that can't be retried safely, e.g. database updates,
401
- it is advisable to annotate it with `@retry(times=0)`.
402
-
403
- This can be used in conjunction with the `@catch` decorator. The `@catch`
404
- decorator will execute a no-op task after all retries have been exhausted,
405
- ensuring that the flow execution can continue.
406
-
407
-
408
- Parameters
409
- ----------
410
- times : int, default 3
411
- Number of times to retry this task.
412
- minutes_between_retries : int, default 2
413
- Number of minutes between retries.
414
- """
415
- ...
416
-
417
- @typing.overload
418
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
419
- ...
420
-
421
- @typing.overload
422
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
423
- ...
424
-
425
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
426
- """
427
- Specifies the number of times the task corresponding
428
- to a step needs to be retried.
429
-
430
- This decorator is useful for handling transient errors, such as networking issues.
431
- If your task contains operations that can't be retried safely, e.g. database updates,
432
- it is advisable to annotate it with `@retry(times=0)`.
433
-
434
- This can be used in conjunction with the `@catch` decorator. The `@catch`
435
- decorator will execute a no-op task after all retries have been exhausted,
436
- ensuring that the flow execution can continue.
437
-
438
-
439
- Parameters
440
- ----------
441
- times : int, default 3
442
- Number of times to retry this task.
443
- minutes_between_retries : int, default 2
444
- Number of minutes between retries.
445
- """
446
- ...
447
-
448
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
449
- """
450
- Decorator that helps cache, version and store models/datasets from huggingface hub.
451
-
452
- > Examples
402
+ > Examples
453
403
 
454
404
  **Usage: creating references of models from huggingface that may be loaded in downstream steps**
455
405
  ```python
@@ -525,246 +475,88 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
525
475
  """
526
476
  ...
527
477
 
528
- def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
478
+ @typing.overload
479
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
529
480
  """
530
- S3 Proxy decorator for routing S3 requests through a local proxy service.
481
+ Specifies environment variables to be set prior to the execution of a step.
531
482
 
532
483
 
533
484
  Parameters
534
485
  ----------
535
- integration_name : str, optional
536
- Name of the S3 proxy integration. If not specified, will use the only
537
- available S3 proxy integration in the namespace (fails if multiple exist).
538
- write_mode : str, optional
539
- The desired behavior during write operations to target (origin) S3 bucket.
540
- allowed options are:
541
- "origin-and-cache" -> write to both the target S3 bucket and local object
542
- storage
543
- "origin" -> only write to the target S3 bucket
544
- "cache" -> only write to the object storage service used for caching
545
- debug : bool, optional
546
- Enable debug logging for proxy operations.
486
+ vars : Dict[str, str], default {}
487
+ Dictionary of environment variables to set.
547
488
  """
548
489
  ...
549
490
 
550
491
  @typing.overload
551
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
552
- """
553
- Decorator prototype for all step decorators. This function gets specialized
554
- and imported for all decorators types by _import_plugin_decorators().
555
- """
492
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
556
493
  ...
557
494
 
558
495
  @typing.overload
559
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
496
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
560
497
  ...
561
498
 
562
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
499
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
563
500
  """
564
- Decorator prototype for all step decorators. This function gets specialized
565
- and imported for all decorators types by _import_plugin_decorators().
501
+ Specifies environment variables to be set prior to the execution of a step.
502
+
503
+
504
+ Parameters
505
+ ----------
506
+ vars : Dict[str, str], default {}
507
+ Dictionary of environment variables to set.
566
508
  """
567
509
  ...
568
510
 
569
511
  @typing.overload
570
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
512
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
571
513
  """
572
- Enables loading / saving of models within a step.
514
+ Enables checkpointing for a step.
573
515
 
574
516
  > Examples
575
- - Saving Models
517
+
518
+ - Saving Checkpoints
519
+
576
520
  ```python
577
- @model
521
+ @checkpoint
578
522
  @step
579
523
  def train(self):
580
- # current.model.save returns a dictionary reference to the model saved
581
- self.my_model = current.model.save(
582
- path_to_my_model,
583
- label="my_model",
584
- metadata={
585
- "epochs": 10,
586
- "batch-size": 32,
587
- "learning-rate": 0.001,
588
- }
589
- )
590
- self.next(self.test)
591
-
592
- @model(load="my_model")
593
- @step
594
- def test(self):
595
- # `current.model.loaded` returns a dictionary of the loaded models
596
- # where the key is the name of the artifact and the value is the path to the model
597
- print(os.listdir(current.model.loaded["my_model"]))
598
- self.next(self.end)
524
+ model = create_model(self.parameters, checkpoint_path = None)
525
+ for i in range(self.epochs):
526
+ # some training logic
527
+ loss = model.train(self.dataset)
528
+ if i % 10 == 0:
529
+ model.save(
530
+ current.checkpoint.directory,
531
+ )
532
+ # saves the contents of the `current.checkpoint.directory` as a checkpoint
533
+ # and returns a reference dictionary to the checkpoint saved in the datastore
534
+ self.latest_checkpoint = current.checkpoint.save(
535
+ name="epoch_checkpoint",
536
+ metadata={
537
+ "epoch": i,
538
+ "loss": loss,
539
+ }
540
+ )
599
541
  ```
600
542
 
601
- - Loading models
543
+ - Using Loaded Checkpoints
544
+
602
545
  ```python
546
+ @retry(times=3)
547
+ @checkpoint
603
548
  @step
604
549
  def train(self):
605
- # current.model.load returns the path to the model loaded
606
- checkpoint_path = current.model.load(
607
- self.checkpoint_key,
608
- )
609
- model_path = current.model.load(
610
- self.model,
611
- )
612
- self.next(self.test)
613
- ```
614
-
615
-
616
- Parameters
617
- ----------
618
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
619
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
620
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
621
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
622
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
623
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
624
-
625
- temp_dir_root : str, default: None
626
- The root directory under which `current.model.loaded` will store loaded models
627
- """
628
- ...
629
-
630
- @typing.overload
631
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
632
- ...
633
-
634
- @typing.overload
635
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
636
- ...
637
-
638
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
639
- """
640
- Enables loading / saving of models within a step.
641
-
642
- > Examples
643
- - Saving Models
644
- ```python
645
- @model
646
- @step
647
- def train(self):
648
- # current.model.save returns a dictionary reference to the model saved
649
- self.my_model = current.model.save(
650
- path_to_my_model,
651
- label="my_model",
652
- metadata={
653
- "epochs": 10,
654
- "batch-size": 32,
655
- "learning-rate": 0.001,
656
- }
657
- )
658
- self.next(self.test)
659
-
660
- @model(load="my_model")
661
- @step
662
- def test(self):
663
- # `current.model.loaded` returns a dictionary of the loaded models
664
- # where the key is the name of the artifact and the value is the path to the model
665
- print(os.listdir(current.model.loaded["my_model"]))
666
- self.next(self.end)
667
- ```
668
-
669
- - Loading models
670
- ```python
671
- @step
672
- def train(self):
673
- # current.model.load returns the path to the model loaded
674
- checkpoint_path = current.model.load(
675
- self.checkpoint_key,
676
- )
677
- model_path = current.model.load(
678
- self.model,
679
- )
680
- self.next(self.test)
681
- ```
682
-
683
-
684
- Parameters
685
- ----------
686
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
687
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
688
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
689
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
690
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
691
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
692
-
693
- temp_dir_root : str, default: None
694
- The root directory under which `current.model.loaded` will store loaded models
695
- """
696
- ...
697
-
698
- @typing.overload
699
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
700
- """
701
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
702
- It exists to make it easier for users to know that this decorator should only be used with
703
- a Neo Cloud like Nebius.
704
- """
705
- ...
706
-
707
- @typing.overload
708
- def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
709
- ...
710
-
711
- def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
712
- """
713
- Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
714
- It exists to make it easier for users to know that this decorator should only be used with
715
- a Neo Cloud like Nebius.
716
- """
717
- ...
718
-
719
- @typing.overload
720
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
721
- """
722
- Enables checkpointing for a step.
723
-
724
- > Examples
725
-
726
- - Saving Checkpoints
727
-
728
- ```python
729
- @checkpoint
730
- @step
731
- def train(self):
732
- model = create_model(self.parameters, checkpoint_path = None)
733
- for i in range(self.epochs):
734
- # some training logic
735
- loss = model.train(self.dataset)
736
- if i % 10 == 0:
737
- model.save(
738
- current.checkpoint.directory,
739
- )
740
- # saves the contents of the `current.checkpoint.directory` as a checkpoint
741
- # and returns a reference dictionary to the checkpoint saved in the datastore
742
- self.latest_checkpoint = current.checkpoint.save(
743
- name="epoch_checkpoint",
744
- metadata={
745
- "epoch": i,
746
- "loss": loss,
747
- }
748
- )
749
- ```
750
-
751
- - Using Loaded Checkpoints
752
-
753
- ```python
754
- @retry(times=3)
755
- @checkpoint
756
- @step
757
- def train(self):
758
- # Assume that the task has restarted and the previous attempt of the task
759
- # saved a checkpoint
760
- checkpoint_path = None
761
- if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
762
- print("Loaded checkpoint from the previous attempt")
763
- checkpoint_path = current.checkpoint.directory
764
-
765
- model = create_model(self.parameters, checkpoint_path = checkpoint_path)
766
- for i in range(self.epochs):
767
- ...
550
+ # Assume that the task has restarted and the previous attempt of the task
551
+ # saved a checkpoint
552
+ checkpoint_path = None
553
+ if current.checkpoint.is_loaded: # Check if a checkpoint is loaded
554
+ print("Loaded checkpoint from the previous attempt")
555
+ checkpoint_path = current.checkpoint.directory
556
+
557
+ model = create_model(self.parameters, checkpoint_path = checkpoint_path)
558
+ for i in range(self.epochs):
559
+ ...
768
560
  ```
769
561
 
770
562
 
@@ -863,108 +655,174 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
863
655
  """
864
656
  ...
865
657
 
866
- @typing.overload
867
- def test_append_card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
868
- """
869
- A simple decorator that demonstrates using CardDecoratorInjector
870
- to inject a card and render simple markdown content.
871
- """
872
- ...
873
-
874
- @typing.overload
875
- def test_append_card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
876
- ...
877
-
878
- def test_append_card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
658
+ def s3_proxy(*, integration_name: typing.Optional[str] = None, write_mode: typing.Optional[str] = None, debug: typing.Optional[bool] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
879
659
  """
880
- A simple decorator that demonstrates using CardDecoratorInjector
881
- to inject a card and render simple markdown content.
660
+ S3 Proxy decorator for routing S3 requests through a local proxy service.
661
+
662
+
663
+ Parameters
664
+ ----------
665
+ integration_name : str, optional
666
+ Name of the S3 proxy integration. If not specified, will use the only
667
+ available S3 proxy integration in the namespace (fails if multiple exist).
668
+ write_mode : str, optional
669
+ The desired behavior during write operations to target (origin) S3 bucket.
670
+ allowed options are:
671
+ "origin-and-cache" -> write to both the target S3 bucket and local object
672
+ storage
673
+ "origin" -> only write to the target S3 bucket
674
+ "cache" -> only write to the object storage service used for caching
675
+ debug : bool, optional
676
+ Enable debug logging for proxy operations.
882
677
  """
883
678
  ...
884
679
 
885
680
  @typing.overload
886
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
681
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
887
682
  """
888
- Specifies the PyPI packages for the step.
683
+ Specifies that the step will success under all circumstances.
889
684
 
890
- Information in this decorator will augment any
891
- attributes set in the `@pyi_base` flow-level decorator. Hence,
892
- you can use `@pypi_base` to set packages required by all
893
- steps and use `@pypi` to specify step-specific overrides.
685
+ The decorator will create an optional artifact, specified by `var`, which
686
+ contains the exception raised. You can use it to detect the presence
687
+ of errors, indicating that all happy-path artifacts produced by the step
688
+ are missing.
894
689
 
895
690
 
896
691
  Parameters
897
692
  ----------
898
- packages : Dict[str, str], default: {}
899
- Packages to use for this step. The key is the name of the package
900
- and the value is the version to use.
901
- python : str, optional, default: None
902
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
903
- that the version used will correspond to the version of the Python interpreter used to start the run.
693
+ var : str, optional, default None
694
+ Name of the artifact in which to store the caught exception.
695
+ If not specified, the exception is not stored.
696
+ print_exception : bool, default True
697
+ Determines whether or not the exception is printed to
698
+ stdout when caught.
904
699
  """
905
700
  ...
906
701
 
907
702
  @typing.overload
908
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
703
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
909
704
  ...
910
705
 
911
706
  @typing.overload
912
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
707
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
913
708
  ...
914
709
 
915
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
710
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
916
711
  """
917
- Specifies the PyPI packages for the step.
712
+ Specifies that the step will success under all circumstances.
918
713
 
919
- Information in this decorator will augment any
920
- attributes set in the `@pyi_base` flow-level decorator. Hence,
921
- you can use `@pypi_base` to set packages required by all
922
- steps and use `@pypi` to specify step-specific overrides.
714
+ The decorator will create an optional artifact, specified by `var`, which
715
+ contains the exception raised. You can use it to detect the presence
716
+ of errors, indicating that all happy-path artifacts produced by the step
717
+ are missing.
923
718
 
924
719
 
925
720
  Parameters
926
721
  ----------
927
- packages : Dict[str, str], default: {}
928
- Packages to use for this step. The key is the name of the package
929
- and the value is the version to use.
930
- python : str, optional, default: None
931
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
932
- that the version used will correspond to the version of the Python interpreter used to start the run.
933
- """
934
- ...
935
-
936
- @typing.overload
937
- def app_deploy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
938
- """
939
- Decorator prototype for all step decorators. This function gets specialized
940
- and imported for all decorators types by _import_plugin_decorators().
722
+ var : str, optional, default None
723
+ Name of the artifact in which to store the caught exception.
724
+ If not specified, the exception is not stored.
725
+ print_exception : bool, default True
726
+ Determines whether or not the exception is printed to
727
+ stdout when caught.
941
728
  """
942
729
  ...
943
730
 
944
731
  @typing.overload
945
- def app_deploy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
946
- ...
947
-
948
- def app_deploy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
949
- """
950
- Decorator prototype for all step decorators. This function gets specialized
951
- and imported for all decorators types by _import_plugin_decorators().
732
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
952
733
  """
953
- ...
954
-
955
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
734
+ Specifies a timeout for your step.
735
+
736
+ This decorator is useful if this step may hang indefinitely.
737
+
738
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
739
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
740
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
741
+
742
+ Note that all the values specified in parameters are added together so if you specify
743
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
744
+
745
+
746
+ Parameters
747
+ ----------
748
+ seconds : int, default 0
749
+ Number of seconds to wait prior to timing out.
750
+ minutes : int, default 0
751
+ Number of minutes to wait prior to timing out.
752
+ hours : int, default 0
753
+ Number of hours to wait prior to timing out.
956
754
  """
957
- Specifies that this step should execute on DGX cloud.
755
+ ...
756
+
757
+ @typing.overload
758
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
759
+ ...
760
+
761
+ @typing.overload
762
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
763
+ ...
764
+
765
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
766
+ """
767
+ Specifies a timeout for your step.
768
+
769
+ This decorator is useful if this step may hang indefinitely.
770
+
771
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
772
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
773
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
774
+
775
+ Note that all the values specified in parameters are added together so if you specify
776
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
958
777
 
959
778
 
960
779
  Parameters
961
780
  ----------
962
- gpu : int
963
- Number of GPUs to use.
964
- gpu_type : str
965
- Type of Nvidia GPU to use.
966
- queue_timeout : int
967
- Time to keep the job in NVCF's queue.
781
+ seconds : int, default 0
782
+ Number of seconds to wait prior to timing out.
783
+ minutes : int, default 0
784
+ Number of minutes to wait prior to timing out.
785
+ hours : int, default 0
786
+ Number of hours to wait prior to timing out.
787
+ """
788
+ ...
789
+
790
+ @typing.overload
791
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
792
+ """
793
+ Specifies secrets to be retrieved and injected as environment variables prior to
794
+ the execution of a step.
795
+
796
+
797
+ Parameters
798
+ ----------
799
+ sources : List[Union[str, Dict[str, Any]]], default: []
800
+ List of secret specs, defining how the secrets are to be retrieved
801
+ role : str, optional, default: None
802
+ Role to use for fetching secrets
803
+ """
804
+ ...
805
+
806
+ @typing.overload
807
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
808
+ ...
809
+
810
+ @typing.overload
811
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
812
+ ...
813
+
814
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
815
+ """
816
+ Specifies secrets to be retrieved and injected as environment variables prior to
817
+ the execution of a step.
818
+
819
+
820
+ Parameters
821
+ ----------
822
+ sources : List[Union[str, Dict[str, Any]]], default: []
823
+ List of secret specs, defining how the secrets are to be retrieved
824
+ role : str, optional, default: None
825
+ Role to use for fetching secrets
968
826
  """
969
827
  ...
970
828
 
@@ -1011,107 +869,403 @@ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy:
1011
869
  """
1012
870
  ...
1013
871
 
872
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
873
+ """
874
+ Specifies that this step should execute on DGX cloud.
875
+
876
+
877
+ Parameters
878
+ ----------
879
+ gpu : int
880
+ Number of GPUs to use.
881
+ gpu_type : str
882
+ Type of Nvidia GPU to use.
883
+ queue_timeout : int
884
+ Time to keep the job in NVCF's queue.
885
+ """
886
+ ...
887
+
888
+ @typing.overload
889
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
890
+ """
891
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
892
+ It exists to make it easier for users to know that this decorator should only be used with
893
+ a Neo Cloud like Nebius.
894
+ """
895
+ ...
896
+
897
+ @typing.overload
898
+ def nebius_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
899
+ ...
900
+
901
+ def nebius_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
902
+ """
903
+ Nebius-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
904
+ It exists to make it easier for users to know that this decorator should only be used with
905
+ a Neo Cloud like Nebius.
906
+ """
907
+ ...
908
+
909
+ @typing.overload
910
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
911
+ """
912
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
913
+ It exists to make it easier for users to know that this decorator should only be used with
914
+ a Neo Cloud like CoreWeave.
915
+ """
916
+ ...
917
+
918
+ @typing.overload
919
+ def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
920
+ ...
921
+
922
+ def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
923
+ """
924
+ CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
925
+ It exists to make it easier for users to know that this decorator should only be used with
926
+ a Neo Cloud like CoreWeave.
927
+ """
928
+ ...
929
+
930
+ @typing.overload
931
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
932
+ """
933
+ Decorator prototype for all step decorators. This function gets specialized
934
+ and imported for all decorators types by _import_plugin_decorators().
935
+ """
936
+ ...
937
+
938
+ @typing.overload
939
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
940
+ ...
941
+
942
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
943
+ """
944
+ Decorator prototype for all step decorators. This function gets specialized
945
+ and imported for all decorators types by _import_plugin_decorators().
946
+ """
947
+ ...
948
+
949
+ @typing.overload
950
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
951
+ """
952
+ Enables loading / saving of models within a step.
953
+
954
+ > Examples
955
+ - Saving Models
956
+ ```python
957
+ @model
958
+ @step
959
+ def train(self):
960
+ # current.model.save returns a dictionary reference to the model saved
961
+ self.my_model = current.model.save(
962
+ path_to_my_model,
963
+ label="my_model",
964
+ metadata={
965
+ "epochs": 10,
966
+ "batch-size": 32,
967
+ "learning-rate": 0.001,
968
+ }
969
+ )
970
+ self.next(self.test)
971
+
972
+ @model(load="my_model")
973
+ @step
974
+ def test(self):
975
+ # `current.model.loaded` returns a dictionary of the loaded models
976
+ # where the key is the name of the artifact and the value is the path to the model
977
+ print(os.listdir(current.model.loaded["my_model"]))
978
+ self.next(self.end)
979
+ ```
980
+
981
+ - Loading models
982
+ ```python
983
+ @step
984
+ def train(self):
985
+ # current.model.load returns the path to the model loaded
986
+ checkpoint_path = current.model.load(
987
+ self.checkpoint_key,
988
+ )
989
+ model_path = current.model.load(
990
+ self.model,
991
+ )
992
+ self.next(self.test)
993
+ ```
994
+
995
+
996
+ Parameters
997
+ ----------
998
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
999
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1000
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1001
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1002
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1003
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1004
+
1005
+ temp_dir_root : str, default: None
1006
+ The root directory under which `current.model.loaded` will store loaded models
1007
+ """
1008
+ ...
1009
+
1010
+ @typing.overload
1011
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1012
+ ...
1013
+
1014
+ @typing.overload
1015
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1016
+ ...
1017
+
1018
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1019
+ """
1020
+ Enables loading / saving of models within a step.
1021
+
1022
+ > Examples
1023
+ - Saving Models
1024
+ ```python
1025
+ @model
1026
+ @step
1027
+ def train(self):
1028
+ # current.model.save returns a dictionary reference to the model saved
1029
+ self.my_model = current.model.save(
1030
+ path_to_my_model,
1031
+ label="my_model",
1032
+ metadata={
1033
+ "epochs": 10,
1034
+ "batch-size": 32,
1035
+ "learning-rate": 0.001,
1036
+ }
1037
+ )
1038
+ self.next(self.test)
1039
+
1040
+ @model(load="my_model")
1041
+ @step
1042
+ def test(self):
1043
+ # `current.model.loaded` returns a dictionary of the loaded models
1044
+ # where the key is the name of the artifact and the value is the path to the model
1045
+ print(os.listdir(current.model.loaded["my_model"]))
1046
+ self.next(self.end)
1047
+ ```
1048
+
1049
+ - Loading models
1050
+ ```python
1051
+ @step
1052
+ def train(self):
1053
+ # current.model.load returns the path to the model loaded
1054
+ checkpoint_path = current.model.load(
1055
+ self.checkpoint_key,
1056
+ )
1057
+ model_path = current.model.load(
1058
+ self.model,
1059
+ )
1060
+ self.next(self.test)
1061
+ ```
1062
+
1063
+
1064
+ Parameters
1065
+ ----------
1066
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
1067
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
1068
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by `current.checkpoint` / `current.model` / `current.huggingface_hub`.
1069
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
1070
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
1071
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1072
+
1073
+ temp_dir_root : str, default: None
1074
+ The root directory under which `current.model.loaded` will store loaded models
1075
+ """
1076
+ ...
1077
+
1078
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1079
+ """
1080
+ Specifies that this step should execute on DGX cloud.
1081
+
1082
+
1083
+ Parameters
1084
+ ----------
1085
+ gpu : int
1086
+ Number of GPUs to use.
1087
+ gpu_type : str
1088
+ Type of Nvidia GPU to use.
1089
+ """
1090
+ ...
1091
+
1092
+ @typing.overload
1093
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1094
+ """
1095
+ Specifies the Conda environment for the step.
1096
+
1097
+ Information in this decorator will augment any
1098
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1099
+ you can use `@conda_base` to set packages required by all
1100
+ steps and use `@conda` to specify step-specific overrides.
1101
+
1102
+
1103
+ Parameters
1104
+ ----------
1105
+ packages : Dict[str, str], default {}
1106
+ Packages to use for this step. The key is the name of the package
1107
+ and the value is the version to use.
1108
+ libraries : Dict[str, str], default {}
1109
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1110
+ python : str, optional, default None
1111
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1112
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1113
+ disabled : bool, default False
1114
+ If set to True, disables @conda.
1115
+ """
1116
+ ...
1117
+
1118
+ @typing.overload
1119
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1120
+ ...
1121
+
1122
+ @typing.overload
1123
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1124
+ ...
1125
+
1126
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1127
+ """
1128
+ Specifies the Conda environment for the step.
1129
+
1130
+ Information in this decorator will augment any
1131
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1132
+ you can use `@conda_base` to set packages required by all
1133
+ steps and use `@conda` to specify step-specific overrides.
1134
+
1135
+
1136
+ Parameters
1137
+ ----------
1138
+ packages : Dict[str, str], default {}
1139
+ Packages to use for this step. The key is the name of the package
1140
+ and the value is the version to use.
1141
+ libraries : Dict[str, str], default {}
1142
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1143
+ python : str, optional, default None
1144
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1145
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1146
+ disabled : bool, default False
1147
+ If set to True, disables @conda.
1148
+ """
1149
+ ...
1150
+
1151
+ @typing.overload
1152
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1153
+ """
1154
+ Internal decorator to support Fast bakery
1155
+ """
1156
+ ...
1157
+
1158
+ @typing.overload
1159
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1160
+ ...
1161
+
1162
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1163
+ """
1164
+ Internal decorator to support Fast bakery
1165
+ """
1166
+ ...
1167
+
1014
1168
  @typing.overload
1015
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1169
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1016
1170
  """
1017
- Specifies environment variables to be set prior to the execution of a step.
1171
+ Specifies the number of times the task corresponding
1172
+ to a step needs to be retried.
1173
+
1174
+ This decorator is useful for handling transient errors, such as networking issues.
1175
+ If your task contains operations that can't be retried safely, e.g. database updates,
1176
+ it is advisable to annotate it with `@retry(times=0)`.
1177
+
1178
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1179
+ decorator will execute a no-op task after all retries have been exhausted,
1180
+ ensuring that the flow execution can continue.
1018
1181
 
1019
1182
 
1020
1183
  Parameters
1021
1184
  ----------
1022
- vars : Dict[str, str], default {}
1023
- Dictionary of environment variables to set.
1185
+ times : int, default 3
1186
+ Number of times to retry this task.
1187
+ minutes_between_retries : int, default 2
1188
+ Number of minutes between retries.
1024
1189
  """
1025
1190
  ...
1026
1191
 
1027
1192
  @typing.overload
1028
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1193
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1029
1194
  ...
1030
1195
 
1031
1196
  @typing.overload
1032
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1197
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1033
1198
  ...
1034
1199
 
1035
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1200
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1036
1201
  """
1037
- Specifies environment variables to be set prior to the execution of a step.
1202
+ Specifies the number of times the task corresponding
1203
+ to a step needs to be retried.
1204
+
1205
+ This decorator is useful for handling transient errors, such as networking issues.
1206
+ If your task contains operations that can't be retried safely, e.g. database updates,
1207
+ it is advisable to annotate it with `@retry(times=0)`.
1208
+
1209
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1210
+ decorator will execute a no-op task after all retries have been exhausted,
1211
+ ensuring that the flow execution can continue.
1038
1212
 
1039
1213
 
1040
1214
  Parameters
1041
1215
  ----------
1042
- vars : Dict[str, str], default {}
1043
- Dictionary of environment variables to set.
1216
+ times : int, default 3
1217
+ Number of times to retry this task.
1218
+ minutes_between_retries : int, default 2
1219
+ Number of minutes between retries.
1044
1220
  """
1045
1221
  ...
1046
1222
 
1047
1223
  @typing.overload
1048
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1224
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1049
1225
  """
1050
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1051
- It exists to make it easier for users to know that this decorator should only be used with
1052
- a Neo Cloud like CoreWeave.
1226
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1227
+
1228
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1229
+
1230
+
1231
+ Parameters
1232
+ ----------
1233
+ type : str, default 'default'
1234
+ Card type.
1235
+ id : str, optional, default None
1236
+ If multiple cards are present, use this id to identify this card.
1237
+ options : Dict[str, Any], default {}
1238
+ Options passed to the card. The contents depend on the card type.
1239
+ timeout : int, default 45
1240
+ Interrupt reporting if it takes more than this many seconds.
1053
1241
  """
1054
1242
  ...
1055
1243
 
1056
1244
  @typing.overload
1057
- def coreweave_s3_proxy(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1245
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1058
1246
  ...
1059
1247
 
1060
- def coreweave_s3_proxy(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1061
- """
1062
- CoreWeave-specific S3 Proxy decorator for routing S3 requests through a local proxy service.
1063
- It exists to make it easier for users to know that this decorator should only be used with
1064
- a Neo Cloud like CoreWeave.
1065
- """
1248
+ @typing.overload
1249
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1066
1250
  ...
1067
1251
 
1068
- def vllm(*, model: str, backend: str, openai_api_server: bool, debug: bool, card_refresh_interval: int, max_retries: int, retry_alert_frequency: int, engine_args: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1252
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1069
1253
  """
1070
- This decorator is used to run vllm APIs as Metaflow task sidecars.
1071
-
1072
- User code call
1073
- --------------
1074
- @vllm(
1075
- model="...",
1076
- ...
1077
- )
1078
-
1079
- Valid backend options
1080
- ---------------------
1081
- - 'local': Run as a separate process on the local task machine.
1082
-
1083
- Valid model options
1084
- -------------------
1085
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
1254
+ Creates a human-readable report, a Metaflow Card, after this step completes.
1086
1255
 
1087
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
1088
- If you need multiple models, you must create multiple @vllm decorators.
1256
+ Note that you may add multiple `@card` decorators in a step with different parameters.
1089
1257
 
1090
1258
 
1091
1259
  Parameters
1092
1260
  ----------
1093
- model: str
1094
- HuggingFace model identifier to be served by vLLM.
1095
- backend: str
1096
- Determines where and how to run the vLLM process.
1097
- openai_api_server: bool
1098
- Whether to use OpenAI-compatible API server mode (subprocess) instead of native engine.
1099
- Default is False (uses native engine).
1100
- Set to True for backward compatibility with existing code.
1101
- debug: bool
1102
- Whether to turn on verbose debugging logs.
1103
- card_refresh_interval: int
1104
- Interval in seconds for refreshing the vLLM status card.
1105
- Only used when openai_api_server=True.
1106
- max_retries: int
1107
- Maximum number of retries checking for vLLM server startup.
1108
- Only used when openai_api_server=True.
1109
- retry_alert_frequency: int
1110
- Frequency of alert logs for vLLM server startup retries.
1111
- Only used when openai_api_server=True.
1112
- engine_args : dict
1113
- Additional keyword arguments to pass to the vLLM engine.
1114
- For example, `tensor_parallel_size=2`.
1261
+ type : str, default 'default'
1262
+ Card type.
1263
+ id : str, optional, default None
1264
+ If multiple cards are present, use this id to identify this card.
1265
+ options : Dict[str, Any], default {}
1266
+ Options passed to the card. The contents depend on the card type.
1267
+ timeout : int, default 45
1268
+ Interrupt reporting if it takes more than this many seconds.
1115
1269
  """
1116
1270
  ...
1117
1271
 
@@ -1184,202 +1338,99 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
1184
1338
  Number of CPUs required for this step.
1185
1339
  gpu : int, optional, default None
1186
1340
  Number of GPUs required for this step.
1187
- disk : int, optional, default None
1188
- Disk size (in MB) required for this step. Only applies on Kubernetes.
1189
- memory : int, default 4096
1190
- Memory size (in MB) required for this step.
1191
- shared_memory : int, optional, default None
1192
- The value for the size (in MiB) of the /dev/shm volume for this step.
1193
- This parameter maps to the `--shm-size` option in Docker.
1194
- """
1195
- ...
1196
-
1197
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1198
- """
1199
- Specifies that this step should execute on Kubernetes.
1200
-
1201
-
1202
- Parameters
1203
- ----------
1204
- cpu : int, default 1
1205
- Number of CPUs required for this step. If `@resources` is
1206
- also present, the maximum value from all decorators is used.
1207
- memory : int, default 4096
1208
- Memory size (in MB) required for this step. If
1209
- `@resources` is also present, the maximum value from all decorators is
1210
- used.
1211
- disk : int, default 10240
1212
- Disk size (in MB) required for this step. If
1213
- `@resources` is also present, the maximum value from all decorators is
1214
- used.
1215
- image : str, optional, default None
1216
- Docker image to use when launching on Kubernetes. If not specified, and
1217
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1218
- not, a default Docker image mapping to the current version of Python is used.
1219
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1220
- If given, the imagePullPolicy to be applied to the Docker image of the step.
1221
- image_pull_secrets: List[str], default []
1222
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
1223
- Kubernetes image pull secrets to use when pulling container images
1224
- in Kubernetes.
1225
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1226
- Kubernetes service account to use when launching pod in Kubernetes.
1227
- secrets : List[str], optional, default None
1228
- Kubernetes secrets to use when launching pod in Kubernetes. These
1229
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1230
- in Metaflow configuration.
1231
- node_selector: Union[Dict[str,str], str], optional, default None
1232
- Kubernetes node selector(s) to apply to the pod running the task.
1233
- Can be passed in as a comma separated string of values e.g.
1234
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
1235
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
1236
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1237
- Kubernetes namespace to use when launching pod in Kubernetes.
1238
- gpu : int, optional, default None
1239
- Number of GPUs required for this step. A value of zero implies that
1240
- the scheduled node should not have GPUs.
1241
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1242
- The vendor of the GPUs to be used for this step.
1243
- tolerations : List[Dict[str,str]], default []
1244
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1245
- Kubernetes tolerations to use when launching pod in Kubernetes.
1246
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
1247
- Kubernetes labels to use when launching pod in Kubernetes.
1248
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
1249
- Kubernetes annotations to use when launching pod in Kubernetes.
1250
- use_tmpfs : bool, default False
1251
- This enables an explicit tmpfs mount for this step.
1252
- tmpfs_tempdir : bool, default True
1253
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1254
- tmpfs_size : int, optional, default: None
1255
- The value for the size (in MiB) of the tmpfs mount for this step.
1256
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1257
- memory allocated for this step.
1258
- tmpfs_path : str, optional, default /metaflow_temp
1259
- Path to tmpfs mount for this step.
1260
- persistent_volume_claims : Dict[str, str], optional, default None
1261
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1262
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1263
- shared_memory: int, optional
1264
- Shared memory size (in MiB) required for this step
1265
- port: int, optional
1266
- Port number to specify in the Kubernetes job object
1267
- compute_pool : str, optional, default None
1268
- Compute pool to be used for for this step.
1269
- If not specified, any accessible compute pool within the perimeter is used.
1270
- hostname_resolution_timeout: int, default 10 * 60
1271
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
1272
- Only applicable when @parallel is used.
1273
- qos: str, default: Burstable
1274
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
1275
-
1276
- security_context: Dict[str, Any], optional, default None
1277
- Container security context. Applies to the task container. Allows the following keys:
1278
- - privileged: bool, optional, default None
1279
- - allow_privilege_escalation: bool, optional, default None
1280
- - run_as_user: int, optional, default None
1281
- - run_as_group: int, optional, default None
1282
- - run_as_non_root: bool, optional, default None
1283
- """
1284
- ...
1285
-
1286
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1287
- """
1288
- Specifies that this step should execute on DGX cloud.
1289
-
1290
-
1291
- Parameters
1292
- ----------
1293
- gpu : int
1294
- Number of GPUs to use.
1295
- gpu_type : str
1296
- Type of Nvidia GPU to use.
1297
- """
1298
- ...
1299
-
1300
- @typing.overload
1301
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1302
- """
1303
- Specifies that the step will success under all circumstances.
1304
-
1305
- The decorator will create an optional artifact, specified by `var`, which
1306
- contains the exception raised. You can use it to detect the presence
1307
- of errors, indicating that all happy-path artifacts produced by the step
1308
- are missing.
1309
-
1310
-
1311
- Parameters
1312
- ----------
1313
- var : str, optional, default None
1314
- Name of the artifact in which to store the caught exception.
1315
- If not specified, the exception is not stored.
1316
- print_exception : bool, default True
1317
- Determines whether or not the exception is printed to
1318
- stdout when caught.
1341
+ disk : int, optional, default None
1342
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
1343
+ memory : int, default 4096
1344
+ Memory size (in MB) required for this step.
1345
+ shared_memory : int, optional, default None
1346
+ The value for the size (in MiB) of the /dev/shm volume for this step.
1347
+ This parameter maps to the `--shm-size` option in Docker.
1319
1348
  """
1320
1349
  ...
1321
1350
 
1322
- @typing.overload
1323
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1324
- ...
1325
-
1326
- @typing.overload
1327
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1328
- ...
1329
-
1330
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1351
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1331
1352
  """
1332
- Specifies that the step will success under all circumstances.
1333
-
1334
- The decorator will create an optional artifact, specified by `var`, which
1335
- contains the exception raised. You can use it to detect the presence
1336
- of errors, indicating that all happy-path artifacts produced by the step
1337
- are missing.
1353
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1354
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1355
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1356
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1357
+ starts only after all sensors finish.
1338
1358
 
1339
1359
 
1340
1360
  Parameters
1341
1361
  ----------
1342
- var : str, optional, default None
1343
- Name of the artifact in which to store the caught exception.
1344
- If not specified, the exception is not stored.
1345
- print_exception : bool, default True
1346
- Determines whether or not the exception is printed to
1347
- stdout when caught.
1362
+ timeout : int
1363
+ Time, in seconds before the task times out and fails. (Default: 3600)
1364
+ poke_interval : int
1365
+ Time in seconds that the job should wait in between each try. (Default: 60)
1366
+ mode : str
1367
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1368
+ exponential_backoff : bool
1369
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1370
+ pool : str
1371
+ the slot pool this task should run in,
1372
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1373
+ soft_fail : bool
1374
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1375
+ name : str
1376
+ Name of the sensor on Airflow
1377
+ description : str
1378
+ Description of sensor in the Airflow UI
1379
+ bucket_key : Union[str, List[str]]
1380
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1381
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1382
+ bucket_name : str
1383
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1384
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1385
+ wildcard_match : bool
1386
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1387
+ aws_conn_id : str
1388
+ a reference to the s3 connection on Airflow. (Default: None)
1389
+ verify : bool
1390
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1348
1391
  """
1349
1392
  ...
1350
1393
 
1351
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1394
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1352
1395
  """
1353
- Specifies what flows belong to the same project.
1354
-
1355
- A project-specific namespace is created for all flows that
1356
- use the same `@project(name)`.
1396
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1397
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1357
1398
 
1358
1399
 
1359
1400
  Parameters
1360
1401
  ----------
1402
+ timeout : int
1403
+ Time, in seconds before the task times out and fails. (Default: 3600)
1404
+ poke_interval : int
1405
+ Time in seconds that the job should wait in between each try. (Default: 60)
1406
+ mode : str
1407
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1408
+ exponential_backoff : bool
1409
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1410
+ pool : str
1411
+ the slot pool this task should run in,
1412
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1413
+ soft_fail : bool
1414
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1361
1415
  name : str
1362
- Project name. Make sure that the name is unique amongst all
1363
- projects that use the same production scheduler. The name may
1364
- contain only lowercase alphanumeric characters and underscores.
1365
-
1366
- branch : Optional[str], default None
1367
- The branch to use. If not specified, the branch is set to
1368
- `user.<username>` unless `production` is set to `True`. This can
1369
- also be set on the command line using `--branch` as a top-level option.
1370
- It is an error to specify `branch` in the decorator and on the command line.
1371
-
1372
- production : bool, default False
1373
- Whether or not the branch is the production branch. This can also be set on the
1374
- command line using `--production` as a top-level option. It is an error to specify
1375
- `production` in the decorator and on the command line.
1376
- The project branch name will be:
1377
- - if `branch` is specified:
1378
- - if `production` is True: `prod.<branch>`
1379
- - if `production` is False: `test.<branch>`
1380
- - if `branch` is not specified:
1381
- - if `production` is True: `prod`
1382
- - if `production` is False: `user.<username>`
1416
+ Name of the sensor on Airflow
1417
+ description : str
1418
+ Description of sensor in the Airflow UI
1419
+ external_dag_id : str
1420
+ The dag_id that contains the task you want to wait for.
1421
+ external_task_ids : List[str]
1422
+ The list of task_ids that you want to wait for.
1423
+ If None (default value) the sensor waits for the DAG. (Default: None)
1424
+ allowed_states : List[str]
1425
+ Iterable of allowed states, (Default: ['success'])
1426
+ failed_states : List[str]
1427
+ Iterable of failed or dis-allowed states. (Default: None)
1428
+ execution_delta : datetime.timedelta
1429
+ time difference with the previous execution to look at,
1430
+ the default is the same logical date as the current task or DAG. (Default: None)
1431
+ check_existence: bool
1432
+ Set to True to check if the external task exists or check if
1433
+ the DAG to wait for exists. (Default: True)
1383
1434
  """
1384
1435
  ...
1385
1436
 
@@ -1476,46 +1527,130 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1476
1527
  """
1477
1528
  ...
1478
1529
 
1479
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1530
+ @typing.overload
1531
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1480
1532
  """
1481
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1482
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1533
+ Specifies the Conda environment for all steps of the flow.
1534
+
1535
+ Use `@conda_base` to set common libraries required by all
1536
+ steps and use `@conda` to specify step-specific additions.
1537
+
1538
+
1539
+ Parameters
1540
+ ----------
1541
+ packages : Dict[str, str], default {}
1542
+ Packages to use for this flow. The key is the name of the package
1543
+ and the value is the version to use.
1544
+ libraries : Dict[str, str], default {}
1545
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1546
+ python : str, optional, default None
1547
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1548
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1549
+ disabled : bool, default False
1550
+ If set to True, disables Conda.
1551
+ """
1552
+ ...
1553
+
1554
+ @typing.overload
1555
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1556
+ ...
1557
+
1558
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1559
+ """
1560
+ Specifies the Conda environment for all steps of the flow.
1561
+
1562
+ Use `@conda_base` to set common libraries required by all
1563
+ steps and use `@conda` to specify step-specific additions.
1564
+
1565
+
1566
+ Parameters
1567
+ ----------
1568
+ packages : Dict[str, str], default {}
1569
+ Packages to use for this flow. The key is the name of the package
1570
+ and the value is the version to use.
1571
+ libraries : Dict[str, str], default {}
1572
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1573
+ python : str, optional, default None
1574
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1575
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1576
+ disabled : bool, default False
1577
+ If set to True, disables Conda.
1578
+ """
1579
+ ...
1580
+
1581
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1582
+ """
1583
+ Specifies what flows belong to the same project.
1584
+
1585
+ A project-specific namespace is created for all flows that
1586
+ use the same `@project(name)`.
1483
1587
 
1484
1588
 
1485
1589
  Parameters
1486
1590
  ----------
1487
- timeout : int
1488
- Time, in seconds before the task times out and fails. (Default: 3600)
1489
- poke_interval : int
1490
- Time in seconds that the job should wait in between each try. (Default: 60)
1491
- mode : str
1492
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1493
- exponential_backoff : bool
1494
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1495
- pool : str
1496
- the slot pool this task should run in,
1497
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1498
- soft_fail : bool
1499
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1500
1591
  name : str
1501
- Name of the sensor on Airflow
1502
- description : str
1503
- Description of sensor in the Airflow UI
1504
- external_dag_id : str
1505
- The dag_id that contains the task you want to wait for.
1506
- external_task_ids : List[str]
1507
- The list of task_ids that you want to wait for.
1508
- If None (default value) the sensor waits for the DAG. (Default: None)
1509
- allowed_states : List[str]
1510
- Iterable of allowed states, (Default: ['success'])
1511
- failed_states : List[str]
1512
- Iterable of failed or dis-allowed states. (Default: None)
1513
- execution_delta : datetime.timedelta
1514
- time difference with the previous execution to look at,
1515
- the default is the same logical date as the current task or DAG. (Default: None)
1516
- check_existence: bool
1517
- Set to True to check if the external task exists or check if
1518
- the DAG to wait for exists. (Default: True)
1592
+ Project name. Make sure that the name is unique amongst all
1593
+ projects that use the same production scheduler. The name may
1594
+ contain only lowercase alphanumeric characters and underscores.
1595
+
1596
+ branch : Optional[str], default None
1597
+ The branch to use. If not specified, the branch is set to
1598
+ `user.<username>` unless `production` is set to `True`. This can
1599
+ also be set on the command line using `--branch` as a top-level option.
1600
+ It is an error to specify `branch` in the decorator and on the command line.
1601
+
1602
+ production : bool, default False
1603
+ Whether or not the branch is the production branch. This can also be set on the
1604
+ command line using `--production` as a top-level option. It is an error to specify
1605
+ `production` in the decorator and on the command line.
1606
+ The project branch name will be:
1607
+ - if `branch` is specified:
1608
+ - if `production` is True: `prod.<branch>`
1609
+ - if `production` is False: `test.<branch>`
1610
+ - if `branch` is not specified:
1611
+ - if `production` is True: `prod`
1612
+ - if `production` is False: `user.<username>`
1613
+ """
1614
+ ...
1615
+
1616
+ @typing.overload
1617
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1618
+ """
1619
+ Specifies the PyPI packages for all steps of the flow.
1620
+
1621
+ Use `@pypi_base` to set common packages required by all
1622
+ steps and use `@pypi` to specify step-specific overrides.
1623
+
1624
+ Parameters
1625
+ ----------
1626
+ packages : Dict[str, str], default: {}
1627
+ Packages to use for this flow. The key is the name of the package
1628
+ and the value is the version to use.
1629
+ python : str, optional, default: None
1630
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1631
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1632
+ """
1633
+ ...
1634
+
1635
+ @typing.overload
1636
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1637
+ ...
1638
+
1639
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1640
+ """
1641
+ Specifies the PyPI packages for all steps of the flow.
1642
+
1643
+ Use `@pypi_base` to set common packages required by all
1644
+ steps and use `@pypi` to specify step-specific overrides.
1645
+
1646
+ Parameters
1647
+ ----------
1648
+ packages : Dict[str, str], default: {}
1649
+ Packages to use for this flow. The key is the name of the package
1650
+ and the value is the version to use.
1651
+ python : str, optional, default: None
1652
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1653
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1519
1654
  """
1520
1655
  ...
1521
1656
 
@@ -1633,192 +1768,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1633
1768
  """
1634
1769
  ...
1635
1770
 
1636
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1637
- """
1638
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1639
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1640
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1641
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1642
- starts only after all sensors finish.
1643
-
1644
-
1645
- Parameters
1646
- ----------
1647
- timeout : int
1648
- Time, in seconds before the task times out and fails. (Default: 3600)
1649
- poke_interval : int
1650
- Time in seconds that the job should wait in between each try. (Default: 60)
1651
- mode : str
1652
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1653
- exponential_backoff : bool
1654
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1655
- pool : str
1656
- the slot pool this task should run in,
1657
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1658
- soft_fail : bool
1659
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1660
- name : str
1661
- Name of the sensor on Airflow
1662
- description : str
1663
- Description of sensor in the Airflow UI
1664
- bucket_key : Union[str, List[str]]
1665
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1666
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1667
- bucket_name : str
1668
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1669
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1670
- wildcard_match : bool
1671
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1672
- aws_conn_id : str
1673
- a reference to the s3 connection on Airflow. (Default: None)
1674
- verify : bool
1675
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1676
- """
1677
- ...
1678
-
1679
- @typing.overload
1680
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1681
- """
1682
- Specifies the Conda environment for all steps of the flow.
1683
-
1684
- Use `@conda_base` to set common libraries required by all
1685
- steps and use `@conda` to specify step-specific additions.
1686
-
1687
-
1688
- Parameters
1689
- ----------
1690
- packages : Dict[str, str], default {}
1691
- Packages to use for this flow. The key is the name of the package
1692
- and the value is the version to use.
1693
- libraries : Dict[str, str], default {}
1694
- Supported for backward compatibility. When used with packages, packages will take precedence.
1695
- python : str, optional, default None
1696
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1697
- that the version used will correspond to the version of the Python interpreter used to start the run.
1698
- disabled : bool, default False
1699
- If set to True, disables Conda.
1700
- """
1701
- ...
1702
-
1703
- @typing.overload
1704
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1705
- ...
1706
-
1707
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1708
- """
1709
- Specifies the Conda environment for all steps of the flow.
1710
-
1711
- Use `@conda_base` to set common libraries required by all
1712
- steps and use `@conda` to specify step-specific additions.
1713
-
1714
-
1715
- Parameters
1716
- ----------
1717
- packages : Dict[str, str], default {}
1718
- Packages to use for this flow. The key is the name of the package
1719
- and the value is the version to use.
1720
- libraries : Dict[str, str], default {}
1721
- Supported for backward compatibility. When used with packages, packages will take precedence.
1722
- python : str, optional, default None
1723
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1724
- that the version used will correspond to the version of the Python interpreter used to start the run.
1725
- disabled : bool, default False
1726
- If set to True, disables Conda.
1727
- """
1728
- ...
1729
-
1730
- @typing.overload
1731
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1732
- """
1733
- Specifies the PyPI packages for all steps of the flow.
1734
-
1735
- Use `@pypi_base` to set common packages required by all
1736
- steps and use `@pypi` to specify step-specific overrides.
1737
-
1738
- Parameters
1739
- ----------
1740
- packages : Dict[str, str], default: {}
1741
- Packages to use for this flow. The key is the name of the package
1742
- and the value is the version to use.
1743
- python : str, optional, default: None
1744
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1745
- that the version used will correspond to the version of the Python interpreter used to start the run.
1746
- """
1747
- ...
1748
-
1749
- @typing.overload
1750
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1751
- ...
1752
-
1753
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1754
- """
1755
- Specifies the PyPI packages for all steps of the flow.
1756
-
1757
- Use `@pypi_base` to set common packages required by all
1758
- steps and use `@pypi` to specify step-specific overrides.
1759
-
1760
- Parameters
1761
- ----------
1762
- packages : Dict[str, str], default: {}
1763
- Packages to use for this flow. The key is the name of the package
1764
- and the value is the version to use.
1765
- python : str, optional, default: None
1766
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1767
- that the version used will correspond to the version of the Python interpreter used to start the run.
1768
- """
1769
- ...
1770
-
1771
- @typing.overload
1772
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1773
- """
1774
- Specifies the times when the flow should be run when running on a
1775
- production scheduler.
1776
-
1777
-
1778
- Parameters
1779
- ----------
1780
- hourly : bool, default False
1781
- Run the workflow hourly.
1782
- daily : bool, default True
1783
- Run the workflow daily.
1784
- weekly : bool, default False
1785
- Run the workflow weekly.
1786
- cron : str, optional, default None
1787
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1788
- specified by this expression.
1789
- timezone : str, optional, default None
1790
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1791
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1792
- """
1793
- ...
1794
-
1795
- @typing.overload
1796
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1797
- ...
1798
-
1799
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1800
- """
1801
- Specifies the times when the flow should be run when running on a
1802
- production scheduler.
1803
-
1804
-
1805
- Parameters
1806
- ----------
1807
- hourly : bool, default False
1808
- Run the workflow hourly.
1809
- daily : bool, default True
1810
- Run the workflow daily.
1811
- weekly : bool, default False
1812
- Run the workflow weekly.
1813
- cron : str, optional, default None
1814
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1815
- specified by this expression.
1816
- timezone : str, optional, default None
1817
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1818
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1819
- """
1820
- ...
1821
-
1822
1771
  @typing.overload
1823
1772
  def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1824
1773
  """
@@ -1920,5 +1869,56 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1920
1869
  """
1921
1870
  ...
1922
1871
 
1872
+ @typing.overload
1873
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1874
+ """
1875
+ Specifies the times when the flow should be run when running on a
1876
+ production scheduler.
1877
+
1878
+
1879
+ Parameters
1880
+ ----------
1881
+ hourly : bool, default False
1882
+ Run the workflow hourly.
1883
+ daily : bool, default True
1884
+ Run the workflow daily.
1885
+ weekly : bool, default False
1886
+ Run the workflow weekly.
1887
+ cron : str, optional, default None
1888
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1889
+ specified by this expression.
1890
+ timezone : str, optional, default None
1891
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1892
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1893
+ """
1894
+ ...
1895
+
1896
+ @typing.overload
1897
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1898
+ ...
1899
+
1900
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1901
+ """
1902
+ Specifies the times when the flow should be run when running on a
1903
+ production scheduler.
1904
+
1905
+
1906
+ Parameters
1907
+ ----------
1908
+ hourly : bool, default False
1909
+ Run the workflow hourly.
1910
+ daily : bool, default True
1911
+ Run the workflow daily.
1912
+ weekly : bool, default False
1913
+ Run the workflow weekly.
1914
+ cron : str, optional, default None
1915
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1916
+ specified by this expression.
1917
+ timezone : str, optional, default None
1918
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1919
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1920
+ """
1921
+ ...
1922
+
1923
1923
  pkg_name: str
1924
1924