ob-metaflow-stubs 6.0.3.182rc1__py2.py3-none-any.whl → 6.0.3.183__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. metaflow-stubs/__init__.pyi +667 -667
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +3 -3
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -4
  20. metaflow-stubs/metaflow_current.pyi +102 -102
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +5 -5
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +4 -4
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +3 -3
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  83. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  84. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  85. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  86. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  87. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  88. metaflow-stubs/multicore_utils.pyi +2 -2
  89. metaflow-stubs/ob_internal.pyi +2 -3
  90. metaflow-stubs/parameters.pyi +3 -3
  91. metaflow-stubs/plugins/__init__.pyi +10 -10
  92. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  93. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  94. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  95. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  96. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  97. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  98. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  99. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  100. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  101. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  102. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  103. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  104. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  105. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  106. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  108. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  109. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  110. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  111. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  112. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  114. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  115. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  116. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  117. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  119. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  120. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  121. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  122. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  123. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  124. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  125. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  126. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  127. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  128. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  129. metaflow-stubs/plugins/cards/__init__.pyi +6 -6
  130. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  131. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  132. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -4
  133. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  134. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  135. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  136. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  137. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  138. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  139. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  140. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  141. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  142. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  143. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  144. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  145. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  146. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  147. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  148. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  149. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  150. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  151. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  152. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  153. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  154. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  155. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  156. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  157. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  158. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  159. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  160. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  161. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  162. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  163. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  164. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  165. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  166. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  167. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  168. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  169. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  170. metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
  171. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  172. metaflow-stubs/plugins/perimeters.pyi +2 -2
  173. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  174. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  175. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  176. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  177. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  178. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  179. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  180. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  181. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  182. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  183. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  184. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  185. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  186. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  187. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  188. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  189. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  190. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  191. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  192. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  193. metaflow-stubs/profilers/__init__.pyi +2 -2
  194. metaflow-stubs/pylint_wrapper.pyi +2 -2
  195. metaflow-stubs/runner/__init__.pyi +2 -2
  196. metaflow-stubs/runner/deployer.pyi +28 -28
  197. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  198. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  199. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  200. metaflow-stubs/runner/nbrun.pyi +2 -2
  201. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  202. metaflow-stubs/runner/utils.pyi +4 -4
  203. metaflow-stubs/system/__init__.pyi +2 -2
  204. metaflow-stubs/system/system_logger.pyi +3 -3
  205. metaflow-stubs/system/system_monitor.pyi +2 -2
  206. metaflow-stubs/tagging_util.pyi +2 -2
  207. metaflow-stubs/tuple_util.pyi +2 -2
  208. metaflow-stubs/user_configs/__init__.pyi +2 -2
  209. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  210. metaflow-stubs/user_configs/config_options.pyi +3 -3
  211. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  212. {ob_metaflow_stubs-6.0.3.182rc1.dist-info → ob_metaflow_stubs-6.0.3.183.dist-info}/METADATA +1 -1
  213. ob_metaflow_stubs-6.0.3.183.dist-info/RECORD +216 -0
  214. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +0 -6
  215. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +0 -51
  216. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +0 -65
  217. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +0 -74
  218. ob_metaflow_stubs-6.0.3.182rc1.dist-info/RECORD +0 -220
  219. {ob_metaflow_stubs-6.0.3.182rc1.dist-info → ob_metaflow_stubs-6.0.3.183.dist-info}/WHEEL +0 -0
  220. {ob_metaflow_stubs-6.0.3.182rc1.dist-info → ob_metaflow_stubs-6.0.3.183.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.17.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-06-18T10:06:22.722981 #
3
+ # MF version: 2.15.18.1+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-06-25T00:26:31.628989 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -35,17 +35,17 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
- from . import metaflow_git as metaflow_git
39
- from . import tuple_util as tuple_util
40
38
  from . import cards as cards
41
39
  from . import events as events
40
+ from . import tuple_util as tuple_util
41
+ from . import metaflow_git as metaflow_git
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
48
47
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
48
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
49
49
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
@@ -156,53 +156,71 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
156
156
  ...
157
157
 
158
158
  @typing.overload
159
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
159
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
160
160
  """
161
- Specifies that the step will success under all circumstances.
162
-
163
- The decorator will create an optional artifact, specified by `var`, which
164
- contains the exception raised. You can use it to detect the presence
165
- of errors, indicating that all happy-path artifacts produced by the step
166
- are missing.
161
+ Specifies environment variables to be set prior to the execution of a step.
167
162
 
168
163
 
169
164
  Parameters
170
165
  ----------
171
- var : str, optional, default None
172
- Name of the artifact in which to store the caught exception.
173
- If not specified, the exception is not stored.
174
- print_exception : bool, default True
175
- Determines whether or not the exception is printed to
176
- stdout when caught.
166
+ vars : Dict[str, str], default {}
167
+ Dictionary of environment variables to set.
177
168
  """
178
169
  ...
179
170
 
180
171
  @typing.overload
181
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
172
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
182
173
  ...
183
174
 
184
175
  @typing.overload
185
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
176
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
186
177
  ...
187
178
 
188
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
179
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
189
180
  """
190
- Specifies that the step will success under all circumstances.
191
-
192
- The decorator will create an optional artifact, specified by `var`, which
193
- contains the exception raised. You can use it to detect the presence
194
- of errors, indicating that all happy-path artifacts produced by the step
195
- are missing.
181
+ Specifies environment variables to be set prior to the execution of a step.
196
182
 
197
183
 
198
184
  Parameters
199
185
  ----------
200
- var : str, optional, default None
201
- Name of the artifact in which to store the caught exception.
202
- If not specified, the exception is not stored.
203
- print_exception : bool, default True
204
- Determines whether or not the exception is printed to
205
- stdout when caught.
186
+ vars : Dict[str, str], default {}
187
+ Dictionary of environment variables to set.
188
+ """
189
+ ...
190
+
191
+ @typing.overload
192
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
193
+ """
194
+ Decorator prototype for all step decorators. This function gets specialized
195
+ and imported for all decorators types by _import_plugin_decorators().
196
+ """
197
+ ...
198
+
199
+ @typing.overload
200
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
201
+ ...
202
+
203
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
204
+ """
205
+ Decorator prototype for all step decorators. This function gets specialized
206
+ and imported for all decorators types by _import_plugin_decorators().
207
+ """
208
+ ...
209
+
210
+ @typing.overload
211
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
212
+ """
213
+ Internal decorator to support Fast bakery
214
+ """
215
+ ...
216
+
217
+ @typing.overload
218
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
219
+ ...
220
+
221
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
222
+ """
223
+ Internal decorator to support Fast bakery
206
224
  """
207
225
  ...
208
226
 
@@ -295,6 +313,63 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
295
313
  """
296
314
  ...
297
315
 
316
+ @typing.overload
317
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
318
+ """
319
+ Enables checkpointing for a step.
320
+
321
+
322
+
323
+ Parameters
324
+ ----------
325
+ load_policy : str, default: "fresh"
326
+ The policy for loading the checkpoint. The following policies are supported:
327
+ - "eager": Loads the the latest available checkpoint within the namespace.
328
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
329
+ will be loaded at the start of the task.
330
+ - "none": Do not load any checkpoint
331
+ - "fresh": Loads the lastest checkpoint created within the running Task.
332
+ This mode helps loading checkpoints across various retry attempts of the same task.
333
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
334
+ created within the task will be loaded when the task is retries execution on failure.
335
+
336
+ temp_dir_root : str, default: None
337
+ The root directory under which `current.checkpoint.directory` will be created.
338
+ """
339
+ ...
340
+
341
+ @typing.overload
342
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
343
+ ...
344
+
345
+ @typing.overload
346
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
347
+ ...
348
+
349
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
350
+ """
351
+ Enables checkpointing for a step.
352
+
353
+
354
+
355
+ Parameters
356
+ ----------
357
+ load_policy : str, default: "fresh"
358
+ The policy for loading the checkpoint. The following policies are supported:
359
+ - "eager": Loads the the latest available checkpoint within the namespace.
360
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
361
+ will be loaded at the start of the task.
362
+ - "none": Do not load any checkpoint
363
+ - "fresh": Loads the lastest checkpoint created within the running Task.
364
+ This mode helps loading checkpoints across various retry attempts of the same task.
365
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
366
+ created within the task will be loaded when the task is retries execution on failure.
367
+
368
+ temp_dir_root : str, default: None
369
+ The root directory under which `current.checkpoint.directory` will be created.
370
+ """
371
+ ...
372
+
298
373
  @typing.overload
299
374
  def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
300
375
  """
@@ -345,53 +420,112 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
345
420
  ...
346
421
 
347
422
  @typing.overload
348
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
423
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
349
424
  """
350
- Specifies the PyPI packages for the step.
425
+ Specifies that the step will success under all circumstances.
426
+
427
+ The decorator will create an optional artifact, specified by `var`, which
428
+ contains the exception raised. You can use it to detect the presence
429
+ of errors, indicating that all happy-path artifacts produced by the step
430
+ are missing.
431
+
432
+
433
+ Parameters
434
+ ----------
435
+ var : str, optional, default None
436
+ Name of the artifact in which to store the caught exception.
437
+ If not specified, the exception is not stored.
438
+ print_exception : bool, default True
439
+ Determines whether or not the exception is printed to
440
+ stdout when caught.
441
+ """
442
+ ...
443
+
444
+ @typing.overload
445
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
446
+ ...
447
+
448
+ @typing.overload
449
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
450
+ ...
451
+
452
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
453
+ """
454
+ Specifies that the step will success under all circumstances.
455
+
456
+ The decorator will create an optional artifact, specified by `var`, which
457
+ contains the exception raised. You can use it to detect the presence
458
+ of errors, indicating that all happy-path artifacts produced by the step
459
+ are missing.
460
+
461
+
462
+ Parameters
463
+ ----------
464
+ var : str, optional, default None
465
+ Name of the artifact in which to store the caught exception.
466
+ If not specified, the exception is not stored.
467
+ print_exception : bool, default True
468
+ Determines whether or not the exception is printed to
469
+ stdout when caught.
470
+ """
471
+ ...
472
+
473
+ @typing.overload
474
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
475
+ """
476
+ Specifies the Conda environment for the step.
351
477
 
352
478
  Information in this decorator will augment any
353
- attributes set in the `@pyi_base` flow-level decorator. Hence,
354
- you can use `@pypi_base` to set packages required by all
355
- steps and use `@pypi` to specify step-specific overrides.
479
+ attributes set in the `@conda_base` flow-level decorator. Hence,
480
+ you can use `@conda_base` to set packages required by all
481
+ steps and use `@conda` to specify step-specific overrides.
356
482
 
357
483
 
358
484
  Parameters
359
485
  ----------
360
- packages : Dict[str, str], default: {}
486
+ packages : Dict[str, str], default {}
361
487
  Packages to use for this step. The key is the name of the package
362
488
  and the value is the version to use.
363
- python : str, optional, default: None
489
+ libraries : Dict[str, str], default {}
490
+ Supported for backward compatibility. When used with packages, packages will take precedence.
491
+ python : str, optional, default None
364
492
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
365
493
  that the version used will correspond to the version of the Python interpreter used to start the run.
494
+ disabled : bool, default False
495
+ If set to True, disables @conda.
366
496
  """
367
497
  ...
368
498
 
369
499
  @typing.overload
370
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
500
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
371
501
  ...
372
502
 
373
503
  @typing.overload
374
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
504
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
375
505
  ...
376
506
 
377
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
507
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
378
508
  """
379
- Specifies the PyPI packages for the step.
509
+ Specifies the Conda environment for the step.
380
510
 
381
511
  Information in this decorator will augment any
382
- attributes set in the `@pyi_base` flow-level decorator. Hence,
383
- you can use `@pypi_base` to set packages required by all
384
- steps and use `@pypi` to specify step-specific overrides.
512
+ attributes set in the `@conda_base` flow-level decorator. Hence,
513
+ you can use `@conda_base` to set packages required by all
514
+ steps and use `@conda` to specify step-specific overrides.
385
515
 
386
516
 
387
517
  Parameters
388
518
  ----------
389
- packages : Dict[str, str], default: {}
519
+ packages : Dict[str, str], default {}
390
520
  Packages to use for this step. The key is the name of the package
391
521
  and the value is the version to use.
392
- python : str, optional, default: None
522
+ libraries : Dict[str, str], default {}
523
+ Supported for backward compatibility. When used with packages, packages will take precedence.
524
+ python : str, optional, default None
393
525
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
394
526
  that the version used will correspond to the version of the Python interpreter used to start the run.
527
+ disabled : bool, default False
528
+ If set to True, disables @conda.
395
529
  """
396
530
  ...
397
531
 
@@ -453,59 +587,240 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
453
587
  ...
454
588
 
455
589
  @typing.overload
456
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
590
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
457
591
  """
458
- Enables checkpointing for a step.
592
+ Specifies the PyPI packages for the step.
459
593
 
594
+ Information in this decorator will augment any
595
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
596
+ you can use `@pypi_base` to set packages required by all
597
+ steps and use `@pypi` to specify step-specific overrides.
460
598
 
461
599
 
462
600
  Parameters
463
601
  ----------
464
- load_policy : str, default: "fresh"
465
- The policy for loading the checkpoint. The following policies are supported:
466
- - "eager": Loads the the latest available checkpoint within the namespace.
467
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
468
- will be loaded at the start of the task.
469
- - "none": Do not load any checkpoint
470
- - "fresh": Loads the lastest checkpoint created within the running Task.
471
- This mode helps loading checkpoints across various retry attempts of the same task.
472
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
473
- created within the task will be loaded when the task is retries execution on failure.
474
-
475
- temp_dir_root : str, default: None
476
- The root directory under which `current.checkpoint.directory` will be created.
602
+ packages : Dict[str, str], default: {}
603
+ Packages to use for this step. The key is the name of the package
604
+ and the value is the version to use.
605
+ python : str, optional, default: None
606
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
607
+ that the version used will correspond to the version of the Python interpreter used to start the run.
477
608
  """
478
609
  ...
479
610
 
480
611
  @typing.overload
481
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
612
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
482
613
  ...
483
614
 
484
615
  @typing.overload
485
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
616
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
486
617
  ...
487
618
 
488
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
619
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
489
620
  """
490
- Enables checkpointing for a step.
621
+ Specifies the PyPI packages for the step.
491
622
 
623
+ Information in this decorator will augment any
624
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
625
+ you can use `@pypi_base` to set packages required by all
626
+ steps and use `@pypi` to specify step-specific overrides.
492
627
 
493
628
 
494
629
  Parameters
495
630
  ----------
496
- load_policy : str, default: "fresh"
497
- The policy for loading the checkpoint. The following policies are supported:
498
- - "eager": Loads the the latest available checkpoint within the namespace.
499
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
500
- will be loaded at the start of the task.
501
- - "none": Do not load any checkpoint
502
- - "fresh": Loads the lastest checkpoint created within the running Task.
503
- This mode helps loading checkpoints across various retry attempts of the same task.
504
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
505
- created within the task will be loaded when the task is retries execution on failure.
631
+ packages : Dict[str, str], default: {}
632
+ Packages to use for this step. The key is the name of the package
633
+ and the value is the version to use.
634
+ python : str, optional, default: None
635
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
636
+ that the version used will correspond to the version of the Python interpreter used to start the run.
637
+ """
638
+ ...
639
+
640
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
641
+ """
642
+ Specifies that this step should execute on DGX cloud.
506
643
 
507
- temp_dir_root : str, default: None
508
- The root directory under which `current.checkpoint.directory` will be created.
644
+
645
+ Parameters
646
+ ----------
647
+ gpu : int
648
+ Number of GPUs to use.
649
+ gpu_type : str
650
+ Type of Nvidia GPU to use.
651
+ queue_timeout : int
652
+ Time to keep the job in NVCF's queue.
653
+ """
654
+ ...
655
+
656
+ @typing.overload
657
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
658
+ """
659
+ Specifies a timeout for your step.
660
+
661
+ This decorator is useful if this step may hang indefinitely.
662
+
663
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
664
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
665
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
666
+
667
+ Note that all the values specified in parameters are added together so if you specify
668
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
669
+
670
+
671
+ Parameters
672
+ ----------
673
+ seconds : int, default 0
674
+ Number of seconds to wait prior to timing out.
675
+ minutes : int, default 0
676
+ Number of minutes to wait prior to timing out.
677
+ hours : int, default 0
678
+ Number of hours to wait prior to timing out.
679
+ """
680
+ ...
681
+
682
+ @typing.overload
683
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
684
+ ...
685
+
686
+ @typing.overload
687
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
688
+ ...
689
+
690
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
691
+ """
692
+ Specifies a timeout for your step.
693
+
694
+ This decorator is useful if this step may hang indefinitely.
695
+
696
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
697
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
698
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
699
+
700
+ Note that all the values specified in parameters are added together so if you specify
701
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
702
+
703
+
704
+ Parameters
705
+ ----------
706
+ seconds : int, default 0
707
+ Number of seconds to wait prior to timing out.
708
+ minutes : int, default 0
709
+ Number of minutes to wait prior to timing out.
710
+ hours : int, default 0
711
+ Number of hours to wait prior to timing out.
712
+ """
713
+ ...
714
+
715
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
716
+ """
717
+ Specifies that this step should execute on DGX cloud.
718
+
719
+
720
+ Parameters
721
+ ----------
722
+ gpu : int
723
+ Number of GPUs to use.
724
+ gpu_type : str
725
+ Type of Nvidia GPU to use.
726
+ """
727
+ ...
728
+
729
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
730
+ """
731
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
732
+
733
+ User code call
734
+ --------------
735
+ @ollama(
736
+ models=[...],
737
+ ...
738
+ )
739
+
740
+ Valid backend options
741
+ ---------------------
742
+ - 'local': Run as a separate process on the local task machine.
743
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
744
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
745
+
746
+ Valid model options
747
+ -------------------
748
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
749
+
750
+
751
+ Parameters
752
+ ----------
753
+ models: list[str]
754
+ List of Ollama containers running models in sidecars.
755
+ backend: str
756
+ Determines where and how to run the Ollama process.
757
+ force_pull: bool
758
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
759
+ cache_update_policy: str
760
+ Cache update policy: "auto", "force", or "never".
761
+ force_cache_update: bool
762
+ Simple override for "force" cache update policy.
763
+ debug: bool
764
+ Whether to turn on verbose debugging logs.
765
+ circuit_breaker_config: dict
766
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
767
+ timeout_config: dict
768
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
769
+ """
770
+ ...
771
+
772
+ @typing.overload
773
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
774
+ """
775
+ Specifies the number of times the task corresponding
776
+ to a step needs to be retried.
777
+
778
+ This decorator is useful for handling transient errors, such as networking issues.
779
+ If your task contains operations that can't be retried safely, e.g. database updates,
780
+ it is advisable to annotate it with `@retry(times=0)`.
781
+
782
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
783
+ decorator will execute a no-op task after all retries have been exhausted,
784
+ ensuring that the flow execution can continue.
785
+
786
+
787
+ Parameters
788
+ ----------
789
+ times : int, default 3
790
+ Number of times to retry this task.
791
+ minutes_between_retries : int, default 2
792
+ Number of minutes between retries.
793
+ """
794
+ ...
795
+
796
+ @typing.overload
797
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
798
+ ...
799
+
800
+ @typing.overload
801
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
802
+ ...
803
+
804
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
805
+ """
806
+ Specifies the number of times the task corresponding
807
+ to a step needs to be retried.
808
+
809
+ This decorator is useful for handling transient errors, such as networking issues.
810
+ If your task contains operations that can't be retried safely, e.g. database updates,
811
+ it is advisable to annotate it with `@retry(times=0)`.
812
+
813
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
814
+ decorator will execute a no-op task after all retries have been exhausted,
815
+ ensuring that the flow execution can continue.
816
+
817
+
818
+ Parameters
819
+ ----------
820
+ times : int, default 3
821
+ Number of times to retry this task.
822
+ minutes_between_retries : int, default 2
823
+ Number of minutes between retries.
509
824
  """
510
825
  ...
511
826
 
@@ -544,19 +859,18 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
544
859
  """
545
860
  ...
546
861
 
547
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
862
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
548
863
  """
549
- Specifies that this step should execute on DGX cloud.
864
+ Specifies that this step is used to deploy an instance of the app.
865
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
550
866
 
551
867
 
552
868
  Parameters
553
869
  ----------
554
- gpu : int
870
+ app_port : int
555
871
  Number of GPUs to use.
556
- gpu_type : str
557
- Type of Nvidia GPU to use.
558
- queue_timeout : int
559
- Time to keep the job in NVCF's queue.
872
+ app_name : str
873
+ Name of the app to deploy.
560
874
  """
561
875
  ...
562
876
 
@@ -664,178 +978,57 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
664
978
  """
665
979
  ...
666
980
 
667
- @typing.overload
668
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
981
+ def vllm(*, model: str, backend: str, debug: bool, kwargs: typing.Any) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
669
982
  """
670
- Specifies the number of times the task corresponding
671
- to a step needs to be retried.
983
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
672
984
 
673
- This decorator is useful for handling transient errors, such as networking issues.
674
- If your task contains operations that can't be retried safely, e.g. database updates,
675
- it is advisable to annotate it with `@retry(times=0)`.
985
+ User code call
986
+ --------------
987
+ @vllm(
988
+ model="...",
989
+ ...
990
+ )
676
991
 
677
- This can be used in conjunction with the `@catch` decorator. The `@catch`
678
- decorator will execute a no-op task after all retries have been exhausted,
679
- ensuring that the flow execution can continue.
992
+ Valid backend options
993
+ ---------------------
994
+ - 'local': Run as a separate process on the local task machine.
995
+
996
+ Valid model options
997
+ -------------------
998
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
999
+
1000
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
1001
+ If you need multiple models, you must create multiple @vllm decorators.
680
1002
 
681
1003
 
682
1004
  Parameters
683
1005
  ----------
684
- times : int, default 3
685
- Number of times to retry this task.
686
- minutes_between_retries : int, default 2
687
- Number of minutes between retries.
688
- """
689
- ...
690
-
691
- @typing.overload
692
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
693
- ...
694
-
695
- @typing.overload
696
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
697
- ...
698
-
699
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
700
- """
701
- Specifies the number of times the task corresponding
702
- to a step needs to be retried.
703
-
704
- This decorator is useful for handling transient errors, such as networking issues.
705
- If your task contains operations that can't be retried safely, e.g. database updates,
706
- it is advisable to annotate it with `@retry(times=0)`.
707
-
708
- This can be used in conjunction with the `@catch` decorator. The `@catch`
709
- decorator will execute a no-op task after all retries have been exhausted,
710
- ensuring that the flow execution can continue.
711
-
712
-
713
- Parameters
714
- ----------
715
- times : int, default 3
716
- Number of times to retry this task.
717
- minutes_between_retries : int, default 2
718
- Number of minutes between retries.
719
- """
720
- ...
721
-
722
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
723
- """
724
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
725
-
726
- User code call
727
- --------------
728
- @ollama(
729
- models=[...],
730
- ...
731
- )
732
-
733
- Valid backend options
734
- ---------------------
735
- - 'local': Run as a separate process on the local task machine.
736
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
737
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
738
-
739
- Valid model options
740
- -------------------
741
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
742
-
743
-
744
- Parameters
745
- ----------
746
- models: list[str]
747
- List of Ollama containers running models in sidecars.
1006
+ model: str
1007
+ HuggingFace model identifier to be served by vLLM.
748
1008
  backend: str
749
- Determines where and how to run the Ollama process.
750
- force_pull: bool
751
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
752
- cache_update_policy: str
753
- Cache update policy: "auto", "force", or "never".
754
- force_cache_update: bool
755
- Simple override for "force" cache update policy.
1009
+ Determines where and how to run the vLLM process.
756
1010
  debug: bool
757
1011
  Whether to turn on verbose debugging logs.
758
- circuit_breaker_config: dict
759
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
760
- timeout_config: dict
761
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
762
- """
763
- ...
764
-
765
- @typing.overload
766
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
767
- """
768
- Specifies a timeout for your step.
769
-
770
- This decorator is useful if this step may hang indefinitely.
771
-
772
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
773
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
774
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
775
-
776
- Note that all the values specified in parameters are added together so if you specify
777
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
778
-
779
-
780
- Parameters
781
- ----------
782
- seconds : int, default 0
783
- Number of seconds to wait prior to timing out.
784
- minutes : int, default 0
785
- Number of minutes to wait prior to timing out.
786
- hours : int, default 0
787
- Number of hours to wait prior to timing out.
788
- """
789
- ...
790
-
791
- @typing.overload
792
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
793
- ...
794
-
795
- @typing.overload
796
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
797
- ...
798
-
799
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
800
- """
801
- Specifies a timeout for your step.
802
-
803
- This decorator is useful if this step may hang indefinitely.
804
-
805
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
806
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
807
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
808
-
809
- Note that all the values specified in parameters are added together so if you specify
810
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
811
-
812
-
813
- Parameters
814
- ----------
815
- seconds : int, default 0
816
- Number of seconds to wait prior to timing out.
817
- minutes : int, default 0
818
- Number of minutes to wait prior to timing out.
819
- hours : int, default 0
820
- Number of hours to wait prior to timing out.
1012
+ kwargs : Any
1013
+ Any other keyword arguments are passed directly to the vLLM engine.
1014
+ This allows for flexible configuration of vLLM server settings.
1015
+ For example, `tensor_parallel_size=2`.
821
1016
  """
822
1017
  ...
823
1018
 
824
1019
  @typing.overload
825
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1020
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
826
1021
  """
827
- Specifies the Conda environment for the step.
1022
+ Specifies the Conda environment for all steps of the flow.
828
1023
 
829
- Information in this decorator will augment any
830
- attributes set in the `@conda_base` flow-level decorator. Hence,
831
- you can use `@conda_base` to set packages required by all
832
- steps and use `@conda` to specify step-specific overrides.
1024
+ Use `@conda_base` to set common libraries required by all
1025
+ steps and use `@conda` to specify step-specific additions.
833
1026
 
834
1027
 
835
1028
  Parameters
836
1029
  ----------
837
1030
  packages : Dict[str, str], default {}
838
- Packages to use for this step. The key is the name of the package
1031
+ Packages to use for this flow. The key is the name of the package
839
1032
  and the value is the version to use.
840
1033
  libraries : Dict[str, str], default {}
841
1034
  Supported for backward compatibility. When used with packages, packages will take precedence.
@@ -843,32 +1036,26 @@ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, s
843
1036
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
844
1037
  that the version used will correspond to the version of the Python interpreter used to start the run.
845
1038
  disabled : bool, default False
846
- If set to True, disables @conda.
1039
+ If set to True, disables Conda.
847
1040
  """
848
1041
  ...
849
1042
 
850
1043
  @typing.overload
851
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
852
- ...
853
-
854
- @typing.overload
855
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1044
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
856
1045
  ...
857
1046
 
858
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1047
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
859
1048
  """
860
- Specifies the Conda environment for the step.
1049
+ Specifies the Conda environment for all steps of the flow.
861
1050
 
862
- Information in this decorator will augment any
863
- attributes set in the `@conda_base` flow-level decorator. Hence,
864
- you can use `@conda_base` to set packages required by all
865
- steps and use `@conda` to specify step-specific overrides.
1051
+ Use `@conda_base` to set common libraries required by all
1052
+ steps and use `@conda` to specify step-specific additions.
866
1053
 
867
1054
 
868
1055
  Parameters
869
1056
  ----------
870
1057
  packages : Dict[str, str], default {}
871
- Packages to use for this step. The key is the name of the package
1058
+ Packages to use for this flow. The key is the name of the package
872
1059
  and the value is the version to use.
873
1060
  libraries : Dict[str, str], default {}
874
1061
  Supported for backward compatibility. When used with packages, packages will take precedence.
@@ -876,143 +1063,200 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
876
1063
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
877
1064
  that the version used will correspond to the version of the Python interpreter used to start the run.
878
1065
  disabled : bool, default False
879
- If set to True, disables @conda.
880
- """
881
- ...
882
-
883
- @typing.overload
884
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
885
- """
886
- Decorator prototype for all step decorators. This function gets specialized
887
- and imported for all decorators types by _import_plugin_decorators().
888
- """
889
- ...
890
-
891
- @typing.overload
892
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
893
- ...
894
-
895
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
896
- """
897
- Decorator prototype for all step decorators. This function gets specialized
898
- and imported for all decorators types by _import_plugin_decorators().
1066
+ If set to True, disables Conda.
899
1067
  """
900
1068
  ...
901
1069
 
902
1070
  @typing.overload
903
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1071
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
904
1072
  """
905
- Specifies environment variables to be set prior to the execution of a step.
1073
+ Specifies the times when the flow should be run when running on a
1074
+ production scheduler.
906
1075
 
907
1076
 
908
1077
  Parameters
909
1078
  ----------
910
- vars : Dict[str, str], default {}
911
- Dictionary of environment variables to set.
1079
+ hourly : bool, default False
1080
+ Run the workflow hourly.
1081
+ daily : bool, default True
1082
+ Run the workflow daily.
1083
+ weekly : bool, default False
1084
+ Run the workflow weekly.
1085
+ cron : str, optional, default None
1086
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1087
+ specified by this expression.
1088
+ timezone : str, optional, default None
1089
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1090
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
912
1091
  """
913
1092
  ...
914
1093
 
915
1094
  @typing.overload
916
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
917
- ...
918
-
919
- @typing.overload
920
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1095
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
921
1096
  ...
922
1097
 
923
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1098
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
924
1099
  """
925
- Specifies environment variables to be set prior to the execution of a step.
1100
+ Specifies the times when the flow should be run when running on a
1101
+ production scheduler.
926
1102
 
927
1103
 
928
1104
  Parameters
929
1105
  ----------
930
- vars : Dict[str, str], default {}
931
- Dictionary of environment variables to set.
1106
+ hourly : bool, default False
1107
+ Run the workflow hourly.
1108
+ daily : bool, default True
1109
+ Run the workflow daily.
1110
+ weekly : bool, default False
1111
+ Run the workflow weekly.
1112
+ cron : str, optional, default None
1113
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1114
+ specified by this expression.
1115
+ timezone : str, optional, default None
1116
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1117
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
932
1118
  """
933
1119
  ...
934
1120
 
935
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1121
+ @typing.overload
1122
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
936
1123
  """
937
- Specifies that this step is used to deploy an instance of the app.
938
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
1124
+ Specifies the PyPI packages for all steps of the flow.
939
1125
 
1126
+ Use `@pypi_base` to set common packages required by all
1127
+ steps and use `@pypi` to specify step-specific overrides.
940
1128
 
941
1129
  Parameters
942
1130
  ----------
943
- app_port : int
944
- Number of GPUs to use.
945
- app_name : str
946
- Name of the app to deploy.
1131
+ packages : Dict[str, str], default: {}
1132
+ Packages to use for this flow. The key is the name of the package
1133
+ and the value is the version to use.
1134
+ python : str, optional, default: None
1135
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1136
+ that the version used will correspond to the version of the Python interpreter used to start the run.
947
1137
  """
948
1138
  ...
949
1139
 
950
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1140
+ @typing.overload
1141
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1142
+ ...
1143
+
1144
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
951
1145
  """
952
- Specifies that this step should execute on DGX cloud.
1146
+ Specifies the PyPI packages for all steps of the flow.
953
1147
 
1148
+ Use `@pypi_base` to set common packages required by all
1149
+ steps and use `@pypi` to specify step-specific overrides.
954
1150
 
955
1151
  Parameters
956
1152
  ----------
957
- gpu : int
958
- Number of GPUs to use.
959
- gpu_type : str
960
- Type of Nvidia GPU to use.
1153
+ packages : Dict[str, str], default: {}
1154
+ Packages to use for this flow. The key is the name of the package
1155
+ and the value is the version to use.
1156
+ python : str, optional, default: None
1157
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1158
+ that the version used will correspond to the version of the Python interpreter used to start the run.
961
1159
  """
962
1160
  ...
963
1161
 
964
- def vllm(*, model: str, backend: str, debug: bool, kwargs: typing.Any) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1162
+ @typing.overload
1163
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
965
1164
  """
966
- This decorator is used to run vllm APIs as Metaflow task sidecars.
1165
+ Specifies the flow(s) that this flow depends on.
967
1166
 
968
- User code call
969
- --------------
970
- @vllm(
971
- model="...",
972
- ...
973
- )
1167
+ ```
1168
+ @trigger_on_finish(flow='FooFlow')
1169
+ ```
1170
+ or
1171
+ ```
1172
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1173
+ ```
1174
+ This decorator respects the @project decorator and triggers the flow
1175
+ when upstream runs within the same namespace complete successfully
974
1176
 
975
- Valid backend options
976
- ---------------------
977
- - 'local': Run as a separate process on the local task machine.
1177
+ Additionally, you can specify project aware upstream flow dependencies
1178
+ by specifying the fully qualified project_flow_name.
1179
+ ```
1180
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1181
+ ```
1182
+ or
1183
+ ```
1184
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1185
+ ```
978
1186
 
979
- Valid model options
980
- -------------------
981
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
1187
+ You can also specify just the project or project branch (other values will be
1188
+ inferred from the current project or project branch):
1189
+ ```
1190
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1191
+ ```
982
1192
 
983
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
984
- If you need multiple models, you must create multiple @vllm decorators.
1193
+ Note that `branch` is typically one of:
1194
+ - `prod`
1195
+ - `user.bob`
1196
+ - `test.my_experiment`
1197
+ - `prod.staging`
985
1198
 
986
1199
 
987
1200
  Parameters
988
1201
  ----------
989
- model: str
990
- HuggingFace model identifier to be served by vLLM.
991
- backend: str
992
- Determines where and how to run the vLLM process.
993
- debug: bool
994
- Whether to turn on verbose debugging logs.
995
- kwargs : Any
996
- Any other keyword arguments are passed directly to the vLLM engine.
997
- This allows for flexible configuration of vLLM server settings.
998
- For example, `tensor_parallel_size=2`.
999
- """
1000
- ...
1001
-
1002
- @typing.overload
1003
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1004
- """
1005
- Internal decorator to support Fast bakery
1202
+ flow : Union[str, Dict[str, str]], optional, default None
1203
+ Upstream flow dependency for this flow.
1204
+ flows : List[Union[str, Dict[str, str]]], default []
1205
+ Upstream flow dependencies for this flow.
1206
+ options : Dict[str, Any], default {}
1207
+ Backend-specific configuration for tuning eventing behavior.
1006
1208
  """
1007
1209
  ...
1008
1210
 
1009
1211
  @typing.overload
1010
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1212
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1011
1213
  ...
1012
1214
 
1013
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1215
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1014
1216
  """
1015
- Internal decorator to support Fast bakery
1217
+ Specifies the flow(s) that this flow depends on.
1218
+
1219
+ ```
1220
+ @trigger_on_finish(flow='FooFlow')
1221
+ ```
1222
+ or
1223
+ ```
1224
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1225
+ ```
1226
+ This decorator respects the @project decorator and triggers the flow
1227
+ when upstream runs within the same namespace complete successfully
1228
+
1229
+ Additionally, you can specify project aware upstream flow dependencies
1230
+ by specifying the fully qualified project_flow_name.
1231
+ ```
1232
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1233
+ ```
1234
+ or
1235
+ ```
1236
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1237
+ ```
1238
+
1239
+ You can also specify just the project or project branch (other values will be
1240
+ inferred from the current project or project branch):
1241
+ ```
1242
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1243
+ ```
1244
+
1245
+ Note that `branch` is typically one of:
1246
+ - `prod`
1247
+ - `user.bob`
1248
+ - `test.my_experiment`
1249
+ - `prod.staging`
1250
+
1251
+
1252
+ Parameters
1253
+ ----------
1254
+ flow : Union[str, Dict[str, str]], optional, default None
1255
+ Upstream flow dependency for this flow.
1256
+ flows : List[Union[str, Dict[str, str]]], default []
1257
+ Upstream flow dependencies for this flow.
1258
+ options : Dict[str, Any], default {}
1259
+ Backend-specific configuration for tuning eventing behavior.
1016
1260
  """
1017
1261
  ...
1018
1262
 
@@ -1130,175 +1374,8 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1130
1374
  """
1131
1375
  ...
1132
1376
 
1133
- @typing.overload
1134
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1135
- """
1136
- Specifies the PyPI packages for all steps of the flow.
1137
-
1138
- Use `@pypi_base` to set common packages required by all
1139
- steps and use `@pypi` to specify step-specific overrides.
1140
-
1141
- Parameters
1142
- ----------
1143
- packages : Dict[str, str], default: {}
1144
- Packages to use for this flow. The key is the name of the package
1145
- and the value is the version to use.
1146
- python : str, optional, default: None
1147
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1148
- that the version used will correspond to the version of the Python interpreter used to start the run.
1149
- """
1150
- ...
1151
-
1152
- @typing.overload
1153
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1154
- ...
1155
-
1156
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1157
- """
1158
- Specifies the PyPI packages for all steps of the flow.
1159
-
1160
- Use `@pypi_base` to set common packages required by all
1161
- steps and use `@pypi` to specify step-specific overrides.
1162
-
1163
- Parameters
1164
- ----------
1165
- packages : Dict[str, str], default: {}
1166
- Packages to use for this flow. The key is the name of the package
1167
- and the value is the version to use.
1168
- python : str, optional, default: None
1169
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1170
- that the version used will correspond to the version of the Python interpreter used to start the run.
1171
- """
1172
- ...
1173
-
1174
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1175
- """
1176
- Specifies what flows belong to the same project.
1177
-
1178
- A project-specific namespace is created for all flows that
1179
- use the same `@project(name)`.
1180
-
1181
-
1182
- Parameters
1183
- ----------
1184
- name : str
1185
- Project name. Make sure that the name is unique amongst all
1186
- projects that use the same production scheduler. The name may
1187
- contain only lowercase alphanumeric characters and underscores.
1188
-
1189
- branch : Optional[str], default None
1190
- The branch to use. If not specified, the branch is set to
1191
- `user.<username>` unless `production` is set to `True`. This can
1192
- also be set on the command line using `--branch` as a top-level option.
1193
- It is an error to specify `branch` in the decorator and on the command line.
1194
-
1195
- production : bool, default False
1196
- Whether or not the branch is the production branch. This can also be set on the
1197
- command line using `--production` as a top-level option. It is an error to specify
1198
- `production` in the decorator and on the command line.
1199
- The project branch name will be:
1200
- - if `branch` is specified:
1201
- - if `production` is True: `prod.<branch>`
1202
- - if `production` is False: `test.<branch>`
1203
- - if `branch` is not specified:
1204
- - if `production` is True: `prod`
1205
- - if `production` is False: `user.<username>`
1206
- """
1207
- ...
1208
-
1209
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1210
- """
1211
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1212
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1213
-
1214
-
1215
- Parameters
1216
- ----------
1217
- timeout : int
1218
- Time, in seconds before the task times out and fails. (Default: 3600)
1219
- poke_interval : int
1220
- Time in seconds that the job should wait in between each try. (Default: 60)
1221
- mode : str
1222
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1223
- exponential_backoff : bool
1224
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1225
- pool : str
1226
- the slot pool this task should run in,
1227
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1228
- soft_fail : bool
1229
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1230
- name : str
1231
- Name of the sensor on Airflow
1232
- description : str
1233
- Description of sensor in the Airflow UI
1234
- external_dag_id : str
1235
- The dag_id that contains the task you want to wait for.
1236
- external_task_ids : List[str]
1237
- The list of task_ids that you want to wait for.
1238
- If None (default value) the sensor waits for the DAG. (Default: None)
1239
- allowed_states : List[str]
1240
- Iterable of allowed states, (Default: ['success'])
1241
- failed_states : List[str]
1242
- Iterable of failed or dis-allowed states. (Default: None)
1243
- execution_delta : datetime.timedelta
1244
- time difference with the previous execution to look at,
1245
- the default is the same logical date as the current task or DAG. (Default: None)
1246
- check_existence: bool
1247
- Set to True to check if the external task exists or check if
1248
- the DAG to wait for exists. (Default: True)
1249
- """
1250
- ...
1251
-
1252
- @typing.overload
1253
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1254
- """
1255
- Specifies the event(s) that this flow depends on.
1256
-
1257
- ```
1258
- @trigger(event='foo')
1259
- ```
1260
- or
1261
- ```
1262
- @trigger(events=['foo', 'bar'])
1263
- ```
1264
-
1265
- Additionally, you can specify the parameter mappings
1266
- to map event payload to Metaflow parameters for the flow.
1267
- ```
1268
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1269
- ```
1270
- or
1271
- ```
1272
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1273
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1274
- ```
1275
-
1276
- 'parameters' can also be a list of strings and tuples like so:
1277
- ```
1278
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1279
- ```
1280
- This is equivalent to:
1281
- ```
1282
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1283
- ```
1284
-
1285
-
1286
- Parameters
1287
- ----------
1288
- event : Union[str, Dict[str, Any]], optional, default None
1289
- Event dependency for this flow.
1290
- events : List[Union[str, Dict[str, Any]]], default []
1291
- Events dependency for this flow.
1292
- options : Dict[str, Any], default {}
1293
- Backend-specific configuration for tuning eventing behavior.
1294
- """
1295
- ...
1296
-
1297
- @typing.overload
1298
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1299
- ...
1300
-
1301
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1377
+ @typing.overload
1378
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1302
1379
  """
1303
1380
  Specifies the event(s) that this flow depends on.
1304
1381
 
@@ -1343,205 +1420,93 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1343
1420
  ...
1344
1421
 
1345
1422
  @typing.overload
1346
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1347
- """
1348
- Specifies the Conda environment for all steps of the flow.
1349
-
1350
- Use `@conda_base` to set common libraries required by all
1351
- steps and use `@conda` to specify step-specific additions.
1352
-
1353
-
1354
- Parameters
1355
- ----------
1356
- packages : Dict[str, str], default {}
1357
- Packages to use for this flow. The key is the name of the package
1358
- and the value is the version to use.
1359
- libraries : Dict[str, str], default {}
1360
- Supported for backward compatibility. When used with packages, packages will take precedence.
1361
- python : str, optional, default None
1362
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1363
- that the version used will correspond to the version of the Python interpreter used to start the run.
1364
- disabled : bool, default False
1365
- If set to True, disables Conda.
1366
- """
1367
- ...
1368
-
1369
- @typing.overload
1370
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1371
- ...
1372
-
1373
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1374
- """
1375
- Specifies the Conda environment for all steps of the flow.
1376
-
1377
- Use `@conda_base` to set common libraries required by all
1378
- steps and use `@conda` to specify step-specific additions.
1379
-
1380
-
1381
- Parameters
1382
- ----------
1383
- packages : Dict[str, str], default {}
1384
- Packages to use for this flow. The key is the name of the package
1385
- and the value is the version to use.
1386
- libraries : Dict[str, str], default {}
1387
- Supported for backward compatibility. When used with packages, packages will take precedence.
1388
- python : str, optional, default None
1389
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1390
- that the version used will correspond to the version of the Python interpreter used to start the run.
1391
- disabled : bool, default False
1392
- If set to True, disables Conda.
1393
- """
1423
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1394
1424
  ...
1395
1425
 
1396
- @typing.overload
1397
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1426
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1398
1427
  """
1399
- Specifies the flow(s) that this flow depends on.
1400
-
1401
- ```
1402
- @trigger_on_finish(flow='FooFlow')
1403
- ```
1404
- or
1405
- ```
1406
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1407
- ```
1408
- This decorator respects the @project decorator and triggers the flow
1409
- when upstream runs within the same namespace complete successfully
1428
+ Specifies the event(s) that this flow depends on.
1410
1429
 
1411
- Additionally, you can specify project aware upstream flow dependencies
1412
- by specifying the fully qualified project_flow_name.
1413
1430
  ```
1414
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1431
+ @trigger(event='foo')
1415
1432
  ```
1416
1433
  or
1417
1434
  ```
1418
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1419
- ```
1420
-
1421
- You can also specify just the project or project branch (other values will be
1422
- inferred from the current project or project branch):
1423
- ```
1424
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1435
+ @trigger(events=['foo', 'bar'])
1425
1436
  ```
1426
1437
 
1427
- Note that `branch` is typically one of:
1428
- - `prod`
1429
- - `user.bob`
1430
- - `test.my_experiment`
1431
- - `prod.staging`
1432
-
1433
-
1434
- Parameters
1435
- ----------
1436
- flow : Union[str, Dict[str, str]], optional, default None
1437
- Upstream flow dependency for this flow.
1438
- flows : List[Union[str, Dict[str, str]]], default []
1439
- Upstream flow dependencies for this flow.
1440
- options : Dict[str, Any], default {}
1441
- Backend-specific configuration for tuning eventing behavior.
1442
- """
1443
- ...
1444
-
1445
- @typing.overload
1446
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1447
- ...
1448
-
1449
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1450
- """
1451
- Specifies the flow(s) that this flow depends on.
1452
-
1438
+ Additionally, you can specify the parameter mappings
1439
+ to map event payload to Metaflow parameters for the flow.
1453
1440
  ```
1454
- @trigger_on_finish(flow='FooFlow')
1441
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1455
1442
  ```
1456
1443
  or
1457
1444
  ```
1458
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1445
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1446
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1459
1447
  ```
1460
- This decorator respects the @project decorator and triggers the flow
1461
- when upstream runs within the same namespace complete successfully
1462
1448
 
1463
- Additionally, you can specify project aware upstream flow dependencies
1464
- by specifying the fully qualified project_flow_name.
1465
- ```
1466
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1467
- ```
1468
- or
1449
+ 'parameters' can also be a list of strings and tuples like so:
1469
1450
  ```
1470
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1451
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1471
1452
  ```
1472
-
1473
- You can also specify just the project or project branch (other values will be
1474
- inferred from the current project or project branch):
1453
+ This is equivalent to:
1475
1454
  ```
1476
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1455
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1477
1456
  ```
1478
1457
 
1479
- Note that `branch` is typically one of:
1480
- - `prod`
1481
- - `user.bob`
1482
- - `test.my_experiment`
1483
- - `prod.staging`
1484
-
1485
1458
 
1486
1459
  Parameters
1487
1460
  ----------
1488
- flow : Union[str, Dict[str, str]], optional, default None
1489
- Upstream flow dependency for this flow.
1490
- flows : List[Union[str, Dict[str, str]]], default []
1491
- Upstream flow dependencies for this flow.
1461
+ event : Union[str, Dict[str, Any]], optional, default None
1462
+ Event dependency for this flow.
1463
+ events : List[Union[str, Dict[str, Any]]], default []
1464
+ Events dependency for this flow.
1492
1465
  options : Dict[str, Any], default {}
1493
1466
  Backend-specific configuration for tuning eventing behavior.
1494
1467
  """
1495
1468
  ...
1496
1469
 
1497
- @typing.overload
1498
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1499
- """
1500
- Specifies the times when the flow should be run when running on a
1501
- production scheduler.
1502
-
1503
-
1504
- Parameters
1505
- ----------
1506
- hourly : bool, default False
1507
- Run the workflow hourly.
1508
- daily : bool, default True
1509
- Run the workflow daily.
1510
- weekly : bool, default False
1511
- Run the workflow weekly.
1512
- cron : str, optional, default None
1513
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1514
- specified by this expression.
1515
- timezone : str, optional, default None
1516
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1517
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1518
- """
1519
- ...
1520
-
1521
- @typing.overload
1522
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1523
- ...
1524
-
1525
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1470
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1526
1471
  """
1527
- Specifies the times when the flow should be run when running on a
1528
- production scheduler.
1472
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1473
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1529
1474
 
1530
1475
 
1531
1476
  Parameters
1532
1477
  ----------
1533
- hourly : bool, default False
1534
- Run the workflow hourly.
1535
- daily : bool, default True
1536
- Run the workflow daily.
1537
- weekly : bool, default False
1538
- Run the workflow weekly.
1539
- cron : str, optional, default None
1540
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1541
- specified by this expression.
1542
- timezone : str, optional, default None
1543
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1544
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1478
+ timeout : int
1479
+ Time, in seconds before the task times out and fails. (Default: 3600)
1480
+ poke_interval : int
1481
+ Time in seconds that the job should wait in between each try. (Default: 60)
1482
+ mode : str
1483
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1484
+ exponential_backoff : bool
1485
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1486
+ pool : str
1487
+ the slot pool this task should run in,
1488
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1489
+ soft_fail : bool
1490
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1491
+ name : str
1492
+ Name of the sensor on Airflow
1493
+ description : str
1494
+ Description of sensor in the Airflow UI
1495
+ external_dag_id : str
1496
+ The dag_id that contains the task you want to wait for.
1497
+ external_task_ids : List[str]
1498
+ The list of task_ids that you want to wait for.
1499
+ If None (default value) the sensor waits for the DAG. (Default: None)
1500
+ allowed_states : List[str]
1501
+ Iterable of allowed states, (Default: ['success'])
1502
+ failed_states : List[str]
1503
+ Iterable of failed or dis-allowed states. (Default: None)
1504
+ execution_delta : datetime.timedelta
1505
+ time difference with the previous execution to look at,
1506
+ the default is the same logical date as the current task or DAG. (Default: None)
1507
+ check_existence: bool
1508
+ Set to True to check if the external task exists or check if
1509
+ the DAG to wait for exists. (Default: True)
1545
1510
  """
1546
1511
  ...
1547
1512
 
@@ -1588,5 +1553,40 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1588
1553
  """
1589
1554
  ...
1590
1555
 
1556
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1557
+ """
1558
+ Specifies what flows belong to the same project.
1559
+
1560
+ A project-specific namespace is created for all flows that
1561
+ use the same `@project(name)`.
1562
+
1563
+
1564
+ Parameters
1565
+ ----------
1566
+ name : str
1567
+ Project name. Make sure that the name is unique amongst all
1568
+ projects that use the same production scheduler. The name may
1569
+ contain only lowercase alphanumeric characters and underscores.
1570
+
1571
+ branch : Optional[str], default None
1572
+ The branch to use. If not specified, the branch is set to
1573
+ `user.<username>` unless `production` is set to `True`. This can
1574
+ also be set on the command line using `--branch` as a top-level option.
1575
+ It is an error to specify `branch` in the decorator and on the command line.
1576
+
1577
+ production : bool, default False
1578
+ Whether or not the branch is the production branch. This can also be set on the
1579
+ command line using `--production` as a top-level option. It is an error to specify
1580
+ `production` in the decorator and on the command line.
1581
+ The project branch name will be:
1582
+ - if `branch` is specified:
1583
+ - if `production` is True: `prod.<branch>`
1584
+ - if `production` is False: `test.<branch>`
1585
+ - if `branch` is not specified:
1586
+ - if `production` is True: `prod`
1587
+ - if `production` is False: `user.<username>`
1588
+ """
1589
+ ...
1590
+
1591
1591
  pkg_name: str
1592
1592