ob-metaflow-stubs 6.0.3.181__py2.py3-none-any.whl → 6.0.3.182__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. metaflow-stubs/__init__.pyi +808 -808
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -4
  20. metaflow-stubs/metaflow_current.pyi +90 -90
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +4 -4
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +4 -4
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  83. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  84. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  85. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  86. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  87. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  88. metaflow-stubs/multicore_utils.pyi +2 -2
  89. metaflow-stubs/parameters.pyi +5 -5
  90. metaflow-stubs/plugins/__init__.pyi +13 -13
  91. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  92. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  93. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  94. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  96. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  97. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  98. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  100. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  101. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  102. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  103. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  104. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  105. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  106. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  107. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  108. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  110. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  111. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  112. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  113. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +5 -5
  114. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  116. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  118. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  119. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  120. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  121. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  122. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  123. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  124. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +5 -5
  125. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  126. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  127. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  128. metaflow-stubs/plugins/cards/__init__.pyi +6 -6
  129. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  130. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  131. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -4
  132. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  133. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  134. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  135. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  136. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  137. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  138. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  139. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  140. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  141. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  142. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  143. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  144. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  146. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  147. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  148. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  149. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  150. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  151. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  152. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  154. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  155. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  156. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  157. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +5 -5
  158. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  159. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  160. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  161. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  162. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  163. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  164. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  165. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  166. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  167. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  168. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  169. metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
  170. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  171. metaflow-stubs/plugins/perimeters.pyi +2 -2
  172. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  173. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  174. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  175. metaflow-stubs/plugins/pypi/conda_environment.pyi +6 -6
  176. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  177. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  178. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  179. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  180. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  181. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  182. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  183. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  184. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  185. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  186. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  187. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  188. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  189. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  190. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  191. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  192. metaflow-stubs/profilers/__init__.pyi +2 -2
  193. metaflow-stubs/pylint_wrapper.pyi +2 -2
  194. metaflow-stubs/runner/__init__.pyi +2 -2
  195. metaflow-stubs/runner/deployer.pyi +5 -5
  196. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  197. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  198. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  199. metaflow-stubs/runner/nbrun.pyi +2 -2
  200. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  201. metaflow-stubs/runner/utils.pyi +3 -3
  202. metaflow-stubs/system/__init__.pyi +2 -2
  203. metaflow-stubs/system/system_logger.pyi +2 -2
  204. metaflow-stubs/system/system_monitor.pyi +2 -2
  205. metaflow-stubs/tagging_util.pyi +2 -2
  206. metaflow-stubs/tuple_util.pyi +2 -2
  207. metaflow-stubs/user_configs/__init__.pyi +2 -2
  208. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  209. metaflow-stubs/user_configs/config_options.pyi +4 -4
  210. metaflow-stubs/user_configs/config_parameters.pyi +7 -7
  211. {ob_metaflow_stubs-6.0.3.181.dist-info → ob_metaflow_stubs-6.0.3.182.dist-info}/METADATA +1 -1
  212. ob_metaflow_stubs-6.0.3.182.dist-info/RECORD +215 -0
  213. ob_metaflow_stubs-6.0.3.181.dist-info/RECORD +0 -215
  214. {ob_metaflow_stubs-6.0.3.181.dist-info → ob_metaflow_stubs-6.0.3.182.dist-info}/WHEEL +0 -0
  215. {ob_metaflow_stubs-6.0.3.181.dist-info → ob_metaflow_stubs-6.0.3.182.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.17.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-06-18T08:04:08.962280 #
3
+ # MF version: 2.15.18.1+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-06-19T23:04:39.675489 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -35,9 +35,9 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
+ from . import tuple_util as tuple_util
38
39
  from . import cards as cards
39
40
  from . import events as events
40
- from . import tuple_util as tuple_util
41
41
  from . import metaflow_git as metaflow_git
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
@@ -45,8 +45,8 @@ from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package imp
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
47
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
48
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
49
48
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
49
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
52
52
  from .client.core import get_namespace as get_namespace
@@ -155,179 +155,86 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
155
155
  ...
156
156
 
157
157
  @typing.overload
158
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
158
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
159
159
  """
160
- Specifies that the step will success under all circumstances.
161
-
162
- The decorator will create an optional artifact, specified by `var`, which
163
- contains the exception raised. You can use it to detect the presence
164
- of errors, indicating that all happy-path artifacts produced by the step
165
- are missing.
160
+ Specifies secrets to be retrieved and injected as environment variables prior to
161
+ the execution of a step.
166
162
 
167
163
 
168
164
  Parameters
169
165
  ----------
170
- var : str, optional, default None
171
- Name of the artifact in which to store the caught exception.
172
- If not specified, the exception is not stored.
173
- print_exception : bool, default True
174
- Determines whether or not the exception is printed to
175
- stdout when caught.
166
+ sources : List[Union[str, Dict[str, Any]]], default: []
167
+ List of secret specs, defining how the secrets are to be retrieved
176
168
  """
177
169
  ...
178
170
 
179
171
  @typing.overload
180
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
172
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
181
173
  ...
182
174
 
183
175
  @typing.overload
184
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
185
- ...
186
-
187
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
188
- """
189
- Specifies that the step will success under all circumstances.
190
-
191
- The decorator will create an optional artifact, specified by `var`, which
192
- contains the exception raised. You can use it to detect the presence
193
- of errors, indicating that all happy-path artifacts produced by the step
194
- are missing.
195
-
196
-
197
- Parameters
198
- ----------
199
- var : str, optional, default None
200
- Name of the artifact in which to store the caught exception.
201
- If not specified, the exception is not stored.
202
- print_exception : bool, default True
203
- Determines whether or not the exception is printed to
204
- stdout when caught.
205
- """
176
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
206
177
  ...
207
178
 
208
- @typing.overload
209
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
179
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
210
180
  """
211
- Specifies the number of times the task corresponding
212
- to a step needs to be retried.
213
-
214
- This decorator is useful for handling transient errors, such as networking issues.
215
- If your task contains operations that can't be retried safely, e.g. database updates,
216
- it is advisable to annotate it with `@retry(times=0)`.
217
-
218
- This can be used in conjunction with the `@catch` decorator. The `@catch`
219
- decorator will execute a no-op task after all retries have been exhausted,
220
- ensuring that the flow execution can continue.
181
+ Specifies secrets to be retrieved and injected as environment variables prior to
182
+ the execution of a step.
221
183
 
222
184
 
223
185
  Parameters
224
186
  ----------
225
- times : int, default 3
226
- Number of times to retry this task.
227
- minutes_between_retries : int, default 2
228
- Number of minutes between retries.
187
+ sources : List[Union[str, Dict[str, Any]]], default: []
188
+ List of secret specs, defining how the secrets are to be retrieved
229
189
  """
230
190
  ...
231
191
 
232
192
  @typing.overload
233
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
234
- ...
235
-
236
- @typing.overload
237
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
238
- ...
239
-
240
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
193
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
241
194
  """
242
- Specifies the number of times the task corresponding
243
- to a step needs to be retried.
244
-
245
- This decorator is useful for handling transient errors, such as networking issues.
246
- If your task contains operations that can't be retried safely, e.g. database updates,
247
- it is advisable to annotate it with `@retry(times=0)`.
248
-
249
- This can be used in conjunction with the `@catch` decorator. The `@catch`
250
- decorator will execute a no-op task after all retries have been exhausted,
251
- ensuring that the flow execution can continue.
252
-
195
+ Creates a human-readable report, a Metaflow Card, after this step completes.
253
196
 
254
- Parameters
255
- ----------
256
- times : int, default 3
257
- Number of times to retry this task.
258
- minutes_between_retries : int, default 2
259
- Number of minutes between retries.
260
- """
261
- ...
262
-
263
- @typing.overload
264
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
265
- """
266
- Specifies environment variables to be set prior to the execution of a step.
197
+ Note that you may add multiple `@card` decorators in a step with different parameters.
267
198
 
268
199
 
269
200
  Parameters
270
201
  ----------
271
- vars : Dict[str, str], default {}
272
- Dictionary of environment variables to set.
202
+ type : str, default 'default'
203
+ Card type.
204
+ id : str, optional, default None
205
+ If multiple cards are present, use this id to identify this card.
206
+ options : Dict[str, Any], default {}
207
+ Options passed to the card. The contents depend on the card type.
208
+ timeout : int, default 45
209
+ Interrupt reporting if it takes more than this many seconds.
273
210
  """
274
211
  ...
275
212
 
276
213
  @typing.overload
277
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
214
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
278
215
  ...
279
216
 
280
217
  @typing.overload
281
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
282
- ...
283
-
284
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
285
- """
286
- Specifies environment variables to be set prior to the execution of a step.
287
-
288
-
289
- Parameters
290
- ----------
291
- vars : Dict[str, str], default {}
292
- Dictionary of environment variables to set.
293
- """
218
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
294
219
  ...
295
220
 
296
- def vllm(*, model: str, backend: str, debug: bool, kwargs: typing.Any) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
221
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
297
222
  """
298
- This decorator is used to run vllm APIs as Metaflow task sidecars.
299
-
300
- User code call
301
- --------------
302
- @vllm(
303
- model="...",
304
- ...
305
- )
306
-
307
- Valid backend options
308
- ---------------------
309
- - 'local': Run as a separate process on the local task machine.
310
-
311
- Valid model options
312
- -------------------
313
- Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
223
+ Creates a human-readable report, a Metaflow Card, after this step completes.
314
224
 
315
- NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
316
- If you need multiple models, you must create multiple @vllm decorators.
225
+ Note that you may add multiple `@card` decorators in a step with different parameters.
317
226
 
318
227
 
319
228
  Parameters
320
229
  ----------
321
- model: str
322
- HuggingFace model identifier to be served by vLLM.
323
- backend: str
324
- Determines where and how to run the vLLM process.
325
- debug: bool
326
- Whether to turn on verbose debugging logs.
327
- kwargs : Any
328
- Any other keyword arguments are passed directly to the vLLM engine.
329
- This allows for flexible configuration of vLLM server settings.
330
- For example, `tensor_parallel_size=2`.
230
+ type : str, default 'default'
231
+ Card type.
232
+ id : str, optional, default None
233
+ If multiple cards are present, use this id to identify this card.
234
+ options : Dict[str, Any], default {}
235
+ Options passed to the card. The contents depend on the card type.
236
+ timeout : int, default 45
237
+ Interrupt reporting if it takes more than this many seconds.
331
238
  """
332
239
  ...
333
240
 
@@ -356,286 +263,23 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
356
263
  """
357
264
  ...
358
265
 
359
- @typing.overload
360
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
361
- """
362
- Enables checkpointing for a step.
363
-
364
-
365
-
366
- Parameters
367
- ----------
368
- load_policy : str, default: "fresh"
369
- The policy for loading the checkpoint. The following policies are supported:
370
- - "eager": Loads the the latest available checkpoint within the namespace.
371
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
372
- will be loaded at the start of the task.
373
- - "none": Do not load any checkpoint
374
- - "fresh": Loads the lastest checkpoint created within the running Task.
375
- This mode helps loading checkpoints across various retry attempts of the same task.
376
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
377
- created within the task will be loaded when the task is retries execution on failure.
378
-
379
- temp_dir_root : str, default: None
380
- The root directory under which `current.checkpoint.directory` will be created.
381
- """
382
- ...
383
-
384
- @typing.overload
385
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
386
- ...
387
-
388
- @typing.overload
389
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
390
- ...
391
-
392
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
266
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
393
267
  """
394
- Enables checkpointing for a step.
395
-
268
+ Specifies that this step should execute on DGX cloud.
396
269
 
397
270
 
398
271
  Parameters
399
272
  ----------
400
- load_policy : str, default: "fresh"
401
- The policy for loading the checkpoint. The following policies are supported:
402
- - "eager": Loads the the latest available checkpoint within the namespace.
403
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
404
- will be loaded at the start of the task.
405
- - "none": Do not load any checkpoint
406
- - "fresh": Loads the lastest checkpoint created within the running Task.
407
- This mode helps loading checkpoints across various retry attempts of the same task.
408
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
409
- created within the task will be loaded when the task is retries execution on failure.
410
-
411
- temp_dir_root : str, default: None
412
- The root directory under which `current.checkpoint.directory` will be created.
273
+ gpu : int
274
+ Number of GPUs to use.
275
+ gpu_type : str
276
+ Type of Nvidia GPU to use.
413
277
  """
414
278
  ...
415
279
 
416
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
280
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
417
281
  """
418
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
419
-
420
- User code call
421
- --------------
422
- @ollama(
423
- models=[...],
424
- ...
425
- )
426
-
427
- Valid backend options
428
- ---------------------
429
- - 'local': Run as a separate process on the local task machine.
430
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
431
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
432
-
433
- Valid model options
434
- -------------------
435
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
436
-
437
-
438
- Parameters
439
- ----------
440
- models: list[str]
441
- List of Ollama containers running models in sidecars.
442
- backend: str
443
- Determines where and how to run the Ollama process.
444
- force_pull: bool
445
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
446
- cache_update_policy: str
447
- Cache update policy: "auto", "force", or "never".
448
- force_cache_update: bool
449
- Simple override for "force" cache update policy.
450
- debug: bool
451
- Whether to turn on verbose debugging logs.
452
- circuit_breaker_config: dict
453
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
454
- timeout_config: dict
455
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
456
- """
457
- ...
458
-
459
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
460
- """
461
- Specifies that this step is used to deploy an instance of the app.
462
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
463
-
464
-
465
- Parameters
466
- ----------
467
- app_port : int
468
- Number of GPUs to use.
469
- app_name : str
470
- Name of the app to deploy.
471
- """
472
- ...
473
-
474
- @typing.overload
475
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
476
- """
477
- Decorator prototype for all step decorators. This function gets specialized
478
- and imported for all decorators types by _import_plugin_decorators().
479
- """
480
- ...
481
-
482
- @typing.overload
483
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
484
- ...
485
-
486
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
487
- """
488
- Decorator prototype for all step decorators. This function gets specialized
489
- and imported for all decorators types by _import_plugin_decorators().
490
- """
491
- ...
492
-
493
- @typing.overload
494
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
495
- """
496
- Enables loading / saving of models within a step.
497
-
498
-
499
-
500
- Parameters
501
- ----------
502
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
503
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
504
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
505
- - `current.checkpoint`
506
- - `current.model`
507
- - `current.huggingface_hub`
508
-
509
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
510
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
511
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
512
-
513
- temp_dir_root : str, default: None
514
- The root directory under which `current.model.loaded` will store loaded models
515
- """
516
- ...
517
-
518
- @typing.overload
519
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
520
- ...
521
-
522
- @typing.overload
523
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
524
- ...
525
-
526
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
527
- """
528
- Enables loading / saving of models within a step.
529
-
530
-
531
-
532
- Parameters
533
- ----------
534
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
535
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
536
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
537
- - `current.checkpoint`
538
- - `current.model`
539
- - `current.huggingface_hub`
540
-
541
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
542
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
543
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
544
-
545
- temp_dir_root : str, default: None
546
- The root directory under which `current.model.loaded` will store loaded models
547
- """
548
- ...
549
-
550
- @typing.overload
551
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
552
- """
553
- Specifies secrets to be retrieved and injected as environment variables prior to
554
- the execution of a step.
555
-
556
-
557
- Parameters
558
- ----------
559
- sources : List[Union[str, Dict[str, Any]]], default: []
560
- List of secret specs, defining how the secrets are to be retrieved
561
- """
562
- ...
563
-
564
- @typing.overload
565
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
566
- ...
567
-
568
- @typing.overload
569
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
570
- ...
571
-
572
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
573
- """
574
- Specifies secrets to be retrieved and injected as environment variables prior to
575
- the execution of a step.
576
-
577
-
578
- Parameters
579
- ----------
580
- sources : List[Union[str, Dict[str, Any]]], default: []
581
- List of secret specs, defining how the secrets are to be retrieved
582
- """
583
- ...
584
-
585
- @typing.overload
586
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
587
- """
588
- Specifies the PyPI packages for the step.
589
-
590
- Information in this decorator will augment any
591
- attributes set in the `@pyi_base` flow-level decorator. Hence,
592
- you can use `@pypi_base` to set packages required by all
593
- steps and use `@pypi` to specify step-specific overrides.
594
-
595
-
596
- Parameters
597
- ----------
598
- packages : Dict[str, str], default: {}
599
- Packages to use for this step. The key is the name of the package
600
- and the value is the version to use.
601
- python : str, optional, default: None
602
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
603
- that the version used will correspond to the version of the Python interpreter used to start the run.
604
- """
605
- ...
606
-
607
- @typing.overload
608
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
609
- ...
610
-
611
- @typing.overload
612
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
613
- ...
614
-
615
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
616
- """
617
- Specifies the PyPI packages for the step.
618
-
619
- Information in this decorator will augment any
620
- attributes set in the `@pyi_base` flow-level decorator. Hence,
621
- you can use `@pypi_base` to set packages required by all
622
- steps and use `@pypi` to specify step-specific overrides.
623
-
624
-
625
- Parameters
626
- ----------
627
- packages : Dict[str, str], default: {}
628
- Packages to use for this step. The key is the name of the package
629
- and the value is the version to use.
630
- python : str, optional, default: None
631
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
632
- that the version used will correspond to the version of the Python interpreter used to start the run.
633
- """
634
- ...
635
-
636
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
637
- """
638
- Specifies that this step should execute on Kubernetes.
282
+ Specifies that this step should execute on Kubernetes.
639
283
 
640
284
 
641
285
  Parameters
@@ -722,155 +366,110 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
722
366
  """
723
367
  ...
724
368
 
725
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
369
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
726
370
  """
727
- Specifies that this step should execute on DGX cloud.
371
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
372
+
373
+ User code call
374
+ --------------
375
+ @ollama(
376
+ models=[...],
377
+ ...
378
+ )
379
+
380
+ Valid backend options
381
+ ---------------------
382
+ - 'local': Run as a separate process on the local task machine.
383
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
384
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
385
+
386
+ Valid model options
387
+ -------------------
388
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
728
389
 
729
390
 
730
391
  Parameters
731
392
  ----------
732
- gpu : int
733
- Number of GPUs to use.
734
- gpu_type : str
735
- Type of Nvidia GPU to use.
393
+ models: list[str]
394
+ List of Ollama containers running models in sidecars.
395
+ backend: str
396
+ Determines where and how to run the Ollama process.
397
+ force_pull: bool
398
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
399
+ cache_update_policy: str
400
+ Cache update policy: "auto", "force", or "never".
401
+ force_cache_update: bool
402
+ Simple override for "force" cache update policy.
403
+ debug: bool
404
+ Whether to turn on verbose debugging logs.
405
+ circuit_breaker_config: dict
406
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
407
+ timeout_config: dict
408
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
736
409
  """
737
410
  ...
738
411
 
739
- @typing.overload
740
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
412
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
741
413
  """
742
- Specifies the Conda environment for the step.
743
-
744
- Information in this decorator will augment any
745
- attributes set in the `@conda_base` flow-level decorator. Hence,
746
- you can use `@conda_base` to set packages required by all
747
- steps and use `@conda` to specify step-specific overrides.
414
+ Specifies that this step is used to deploy an instance of the app.
415
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
748
416
 
749
417
 
750
418
  Parameters
751
419
  ----------
752
- packages : Dict[str, str], default {}
753
- Packages to use for this step. The key is the name of the package
754
- and the value is the version to use.
755
- libraries : Dict[str, str], default {}
756
- Supported for backward compatibility. When used with packages, packages will take precedence.
757
- python : str, optional, default None
758
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
759
- that the version used will correspond to the version of the Python interpreter used to start the run.
760
- disabled : bool, default False
761
- If set to True, disables @conda.
420
+ app_port : int
421
+ Number of GPUs to use.
422
+ app_name : str
423
+ Name of the app to deploy.
762
424
  """
763
425
  ...
764
426
 
765
427
  @typing.overload
766
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
767
- ...
768
-
769
- @typing.overload
770
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
771
- ...
772
-
773
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
428
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
774
429
  """
775
- Specifies the Conda environment for the step.
776
-
777
- Information in this decorator will augment any
778
- attributes set in the `@conda_base` flow-level decorator. Hence,
779
- you can use `@conda_base` to set packages required by all
780
- steps and use `@conda` to specify step-specific overrides.
430
+ Specifies environment variables to be set prior to the execution of a step.
781
431
 
782
432
 
783
433
  Parameters
784
434
  ----------
785
- packages : Dict[str, str], default {}
786
- Packages to use for this step. The key is the name of the package
787
- and the value is the version to use.
788
- libraries : Dict[str, str], default {}
789
- Supported for backward compatibility. When used with packages, packages will take precedence.
790
- python : str, optional, default None
791
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
792
- that the version used will correspond to the version of the Python interpreter used to start the run.
793
- disabled : bool, default False
794
- If set to True, disables @conda.
435
+ vars : Dict[str, str], default {}
436
+ Dictionary of environment variables to set.
795
437
  """
796
438
  ...
797
439
 
798
440
  @typing.overload
799
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
441
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
442
+ ...
443
+
444
+ @typing.overload
445
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
446
+ ...
447
+
448
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
800
449
  """
801
- Specifies the resources needed when executing this step.
802
-
803
- Use `@resources` to specify the resource requirements
804
- independently of the specific compute layer (`@batch`, `@kubernetes`).
805
-
806
- You can choose the compute layer on the command line by executing e.g.
807
- ```
808
- python myflow.py run --with batch
809
- ```
810
- or
811
- ```
812
- python myflow.py run --with kubernetes
813
- ```
814
- which executes the flow on the desired system using the
815
- requirements specified in `@resources`.
450
+ Specifies environment variables to be set prior to the execution of a step.
816
451
 
817
452
 
818
453
  Parameters
819
454
  ----------
820
- cpu : int, default 1
821
- Number of CPUs required for this step.
822
- gpu : int, optional, default None
823
- Number of GPUs required for this step.
824
- disk : int, optional, default None
825
- Disk size (in MB) required for this step. Only applies on Kubernetes.
826
- memory : int, default 4096
827
- Memory size (in MB) required for this step.
828
- shared_memory : int, optional, default None
829
- The value for the size (in MiB) of the /dev/shm volume for this step.
830
- This parameter maps to the `--shm-size` option in Docker.
455
+ vars : Dict[str, str], default {}
456
+ Dictionary of environment variables to set.
831
457
  """
832
458
  ...
833
459
 
834
- @typing.overload
835
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
836
- ...
837
-
838
- @typing.overload
839
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
840
- ...
841
-
842
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
460
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
843
461
  """
844
- Specifies the resources needed when executing this step.
845
-
846
- Use `@resources` to specify the resource requirements
847
- independently of the specific compute layer (`@batch`, `@kubernetes`).
848
-
849
- You can choose the compute layer on the command line by executing e.g.
850
- ```
851
- python myflow.py run --with batch
852
- ```
853
- or
854
- ```
855
- python myflow.py run --with kubernetes
856
- ```
857
- which executes the flow on the desired system using the
858
- requirements specified in `@resources`.
462
+ Specifies that this step should execute on DGX cloud.
859
463
 
860
464
 
861
465
  Parameters
862
466
  ----------
863
- cpu : int, default 1
864
- Number of CPUs required for this step.
865
- gpu : int, optional, default None
866
- Number of GPUs required for this step.
867
- disk : int, optional, default None
868
- Disk size (in MB) required for this step. Only applies on Kubernetes.
869
- memory : int, default 4096
870
- Memory size (in MB) required for this step.
871
- shared_memory : int, optional, default None
872
- The value for the size (in MiB) of the /dev/shm volume for this step.
873
- This parameter maps to the `--shm-size` option in Docker.
467
+ gpu : int
468
+ Number of GPUs to use.
469
+ gpu_type : str
470
+ Type of Nvidia GPU to use.
471
+ queue_timeout : int
472
+ Time to keep the job in NVCF's queue.
874
473
  """
875
474
  ...
876
475
 
@@ -934,178 +533,204 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
934
533
  ...
935
534
 
936
535
  @typing.overload
937
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
536
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
938
537
  """
939
- Internal decorator to support Fast bakery
538
+ Specifies that the step will success under all circumstances.
539
+
540
+ The decorator will create an optional artifact, specified by `var`, which
541
+ contains the exception raised. You can use it to detect the presence
542
+ of errors, indicating that all happy-path artifacts produced by the step
543
+ are missing.
544
+
545
+
546
+ Parameters
547
+ ----------
548
+ var : str, optional, default None
549
+ Name of the artifact in which to store the caught exception.
550
+ If not specified, the exception is not stored.
551
+ print_exception : bool, default True
552
+ Determines whether or not the exception is printed to
553
+ stdout when caught.
940
554
  """
941
555
  ...
942
556
 
943
557
  @typing.overload
944
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
558
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
945
559
  ...
946
560
 
947
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
948
- """
949
- Internal decorator to support Fast bakery
950
- """
561
+ @typing.overload
562
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
951
563
  ...
952
564
 
953
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
565
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
954
566
  """
955
- Specifies that this step should execute on DGX cloud.
567
+ Specifies that the step will success under all circumstances.
568
+
569
+ The decorator will create an optional artifact, specified by `var`, which
570
+ contains the exception raised. You can use it to detect the presence
571
+ of errors, indicating that all happy-path artifacts produced by the step
572
+ are missing.
956
573
 
957
574
 
958
575
  Parameters
959
576
  ----------
960
- gpu : int
961
- Number of GPUs to use.
962
- gpu_type : str
963
- Type of Nvidia GPU to use.
964
- queue_timeout : int
965
- Time to keep the job in NVCF's queue.
577
+ var : str, optional, default None
578
+ Name of the artifact in which to store the caught exception.
579
+ If not specified, the exception is not stored.
580
+ print_exception : bool, default True
581
+ Determines whether or not the exception is printed to
582
+ stdout when caught.
966
583
  """
967
584
  ...
968
585
 
969
586
  @typing.overload
970
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
587
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
971
588
  """
972
- Creates a human-readable report, a Metaflow Card, after this step completes.
589
+ Enables loading / saving of models within a step.
973
590
 
974
- Note that you may add multiple `@card` decorators in a step with different parameters.
975
591
 
976
592
 
977
593
  Parameters
978
594
  ----------
979
- type : str, default 'default'
980
- Card type.
981
- id : str, optional, default None
982
- If multiple cards are present, use this id to identify this card.
983
- options : Dict[str, Any], default {}
984
- Options passed to the card. The contents depend on the card type.
985
- timeout : int, default 45
986
- Interrupt reporting if it takes more than this many seconds.
595
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
596
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
597
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
598
+ - `current.checkpoint`
599
+ - `current.model`
600
+ - `current.huggingface_hub`
601
+
602
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
603
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
604
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
605
+
606
+ temp_dir_root : str, default: None
607
+ The root directory under which `current.model.loaded` will store loaded models
987
608
  """
988
609
  ...
989
610
 
990
611
  @typing.overload
991
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
612
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
992
613
  ...
993
614
 
994
615
  @typing.overload
995
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
996
- ...
997
-
998
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
999
- """
1000
- Creates a human-readable report, a Metaflow Card, after this step completes.
1001
-
1002
- Note that you may add multiple `@card` decorators in a step with different parameters.
1003
-
1004
-
1005
- Parameters
1006
- ----------
1007
- type : str, default 'default'
1008
- Card type.
1009
- id : str, optional, default None
1010
- If multiple cards are present, use this id to identify this card.
1011
- options : Dict[str, Any], default {}
1012
- Options passed to the card. The contents depend on the card type.
1013
- timeout : int, default 45
1014
- Interrupt reporting if it takes more than this many seconds.
1015
- """
616
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1016
617
  ...
1017
618
 
1018
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
619
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
1019
620
  """
1020
- Specifies what flows belong to the same project.
621
+ Enables loading / saving of models within a step.
1021
622
 
1022
- A project-specific namespace is created for all flows that
1023
- use the same `@project(name)`.
1024
623
 
1025
624
 
1026
625
  Parameters
1027
626
  ----------
1028
- name : str
1029
- Project name. Make sure that the name is unique amongst all
1030
- projects that use the same production scheduler. The name may
1031
- contain only lowercase alphanumeric characters and underscores.
627
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
628
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
629
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
630
+ - `current.checkpoint`
631
+ - `current.model`
632
+ - `current.huggingface_hub`
1032
633
 
1033
- branch : Optional[str], default None
1034
- The branch to use. If not specified, the branch is set to
1035
- `user.<username>` unless `production` is set to `True`. This can
1036
- also be set on the command line using `--branch` as a top-level option.
1037
- It is an error to specify `branch` in the decorator and on the command line.
634
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
635
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
636
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
1038
637
 
1039
- production : bool, default False
1040
- Whether or not the branch is the production branch. This can also be set on the
1041
- command line using `--production` as a top-level option. It is an error to specify
1042
- `production` in the decorator and on the command line.
1043
- The project branch name will be:
1044
- - if `branch` is specified:
1045
- - if `production` is True: `prod.<branch>`
1046
- - if `production` is False: `test.<branch>`
1047
- - if `branch` is not specified:
1048
- - if `production` is True: `prod`
1049
- - if `production` is False: `user.<username>`
638
+ temp_dir_root : str, default: None
639
+ The root directory under which `current.model.loaded` will store loaded models
1050
640
  """
1051
641
  ...
1052
642
 
1053
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
643
+ @typing.overload
644
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1054
645
  """
1055
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1056
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
646
+ Enables checkpointing for a step.
647
+
1057
648
 
1058
649
 
1059
650
  Parameters
1060
651
  ----------
1061
- timeout : int
1062
- Time, in seconds before the task times out and fails. (Default: 3600)
1063
- poke_interval : int
1064
- Time in seconds that the job should wait in between each try. (Default: 60)
1065
- mode : str
1066
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1067
- exponential_backoff : bool
1068
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1069
- pool : str
1070
- the slot pool this task should run in,
1071
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1072
- soft_fail : bool
1073
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1074
- name : str
1075
- Name of the sensor on Airflow
1076
- description : str
1077
- Description of sensor in the Airflow UI
1078
- external_dag_id : str
1079
- The dag_id that contains the task you want to wait for.
1080
- external_task_ids : List[str]
1081
- The list of task_ids that you want to wait for.
1082
- If None (default value) the sensor waits for the DAG. (Default: None)
1083
- allowed_states : List[str]
1084
- Iterable of allowed states, (Default: ['success'])
1085
- failed_states : List[str]
1086
- Iterable of failed or dis-allowed states. (Default: None)
1087
- execution_delta : datetime.timedelta
1088
- time difference with the previous execution to look at,
1089
- the default is the same logical date as the current task or DAG. (Default: None)
1090
- check_existence: bool
1091
- Set to True to check if the external task exists or check if
1092
- the DAG to wait for exists. (Default: True)
652
+ load_policy : str, default: "fresh"
653
+ The policy for loading the checkpoint. The following policies are supported:
654
+ - "eager": Loads the the latest available checkpoint within the namespace.
655
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
656
+ will be loaded at the start of the task.
657
+ - "none": Do not load any checkpoint
658
+ - "fresh": Loads the lastest checkpoint created within the running Task.
659
+ This mode helps loading checkpoints across various retry attempts of the same task.
660
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
661
+ created within the task will be loaded when the task is retries execution on failure.
662
+
663
+ temp_dir_root : str, default: None
664
+ The root directory under which `current.checkpoint.directory` will be created.
1093
665
  """
1094
666
  ...
1095
667
 
1096
668
  @typing.overload
1097
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
669
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
670
+ ...
671
+
672
+ @typing.overload
673
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
674
+ ...
675
+
676
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
1098
677
  """
1099
- Specifies the Conda environment for all steps of the flow.
678
+ Enables checkpointing for a step.
1100
679
 
1101
- Use `@conda_base` to set common libraries required by all
1102
- steps and use `@conda` to specify step-specific additions.
680
+
681
+
682
+ Parameters
683
+ ----------
684
+ load_policy : str, default: "fresh"
685
+ The policy for loading the checkpoint. The following policies are supported:
686
+ - "eager": Loads the the latest available checkpoint within the namespace.
687
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
688
+ will be loaded at the start of the task.
689
+ - "none": Do not load any checkpoint
690
+ - "fresh": Loads the lastest checkpoint created within the running Task.
691
+ This mode helps loading checkpoints across various retry attempts of the same task.
692
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
693
+ created within the task will be loaded when the task is retries execution on failure.
694
+
695
+ temp_dir_root : str, default: None
696
+ The root directory under which `current.checkpoint.directory` will be created.
697
+ """
698
+ ...
699
+
700
+ @typing.overload
701
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
702
+ """
703
+ Decorator prototype for all step decorators. This function gets specialized
704
+ and imported for all decorators types by _import_plugin_decorators().
705
+ """
706
+ ...
707
+
708
+ @typing.overload
709
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
710
+ ...
711
+
712
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
713
+ """
714
+ Decorator prototype for all step decorators. This function gets specialized
715
+ and imported for all decorators types by _import_plugin_decorators().
716
+ """
717
+ ...
718
+
719
+ @typing.overload
720
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
721
+ """
722
+ Specifies the Conda environment for the step.
723
+
724
+ Information in this decorator will augment any
725
+ attributes set in the `@conda_base` flow-level decorator. Hence,
726
+ you can use `@conda_base` to set packages required by all
727
+ steps and use `@conda` to specify step-specific overrides.
1103
728
 
1104
729
 
1105
730
  Parameters
1106
731
  ----------
1107
732
  packages : Dict[str, str], default {}
1108
- Packages to use for this flow. The key is the name of the package
733
+ Packages to use for this step. The key is the name of the package
1109
734
  and the value is the version to use.
1110
735
  libraries : Dict[str, str], default {}
1111
736
  Supported for backward compatibility. When used with packages, packages will take precedence.
@@ -1113,26 +738,32 @@ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[s
1113
738
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1114
739
  that the version used will correspond to the version of the Python interpreter used to start the run.
1115
740
  disabled : bool, default False
1116
- If set to True, disables Conda.
741
+ If set to True, disables @conda.
1117
742
  """
1118
743
  ...
1119
744
 
1120
745
  @typing.overload
1121
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
746
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1122
747
  ...
1123
748
 
1124
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
749
+ @typing.overload
750
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
751
+ ...
752
+
753
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1125
754
  """
1126
- Specifies the Conda environment for all steps of the flow.
755
+ Specifies the Conda environment for the step.
1127
756
 
1128
- Use `@conda_base` to set common libraries required by all
1129
- steps and use `@conda` to specify step-specific additions.
757
+ Information in this decorator will augment any
758
+ attributes set in the `@conda_base` flow-level decorator. Hence,
759
+ you can use `@conda_base` to set packages required by all
760
+ steps and use `@conda` to specify step-specific overrides.
1130
761
 
1131
762
 
1132
763
  Parameters
1133
764
  ----------
1134
765
  packages : Dict[str, str], default {}
1135
- Packages to use for this flow. The key is the name of the package
766
+ Packages to use for this step. The key is the name of the package
1136
767
  and the value is the version to use.
1137
768
  libraries : Dict[str, str], default {}
1138
769
  Supported for backward compatibility. When used with packages, packages will take precedence.
@@ -1140,151 +771,290 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1140
771
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1141
772
  that the version used will correspond to the version of the Python interpreter used to start the run.
1142
773
  disabled : bool, default False
1143
- If set to True, disables Conda.
774
+ If set to True, disables @conda.
775
+ """
776
+ ...
777
+
778
+ def vllm(*, model: str, backend: str, debug: bool, kwargs: typing.Any) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
779
+ """
780
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
781
+
782
+ User code call
783
+ --------------
784
+ @vllm(
785
+ model="...",
786
+ ...
787
+ )
788
+
789
+ Valid backend options
790
+ ---------------------
791
+ - 'local': Run as a separate process on the local task machine.
792
+
793
+ Valid model options
794
+ -------------------
795
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
796
+
797
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
798
+ If you need multiple models, you must create multiple @vllm decorators.
799
+
800
+
801
+ Parameters
802
+ ----------
803
+ model: str
804
+ HuggingFace model identifier to be served by vLLM.
805
+ backend: str
806
+ Determines where and how to run the vLLM process.
807
+ debug: bool
808
+ Whether to turn on verbose debugging logs.
809
+ kwargs : Any
810
+ Any other keyword arguments are passed directly to the vLLM engine.
811
+ This allows for flexible configuration of vLLM server settings.
812
+ For example, `tensor_parallel_size=2`.
1144
813
  """
1145
814
  ...
1146
815
 
1147
816
  @typing.overload
1148
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
817
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1149
818
  """
1150
- Specifies the event(s) that this flow depends on.
819
+ Specifies the PyPI packages for the step.
1151
820
 
1152
- ```
1153
- @trigger(event='foo')
1154
- ```
1155
- or
1156
- ```
1157
- @trigger(events=['foo', 'bar'])
1158
- ```
821
+ Information in this decorator will augment any
822
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
823
+ you can use `@pypi_base` to set packages required by all
824
+ steps and use `@pypi` to specify step-specific overrides.
1159
825
 
1160
- Additionally, you can specify the parameter mappings
1161
- to map event payload to Metaflow parameters for the flow.
1162
- ```
1163
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1164
- ```
1165
- or
1166
- ```
1167
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1168
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1169
- ```
1170
826
 
1171
- 'parameters' can also be a list of strings and tuples like so:
1172
- ```
1173
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1174
- ```
1175
- This is equivalent to:
1176
- ```
1177
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1178
- ```
827
+ Parameters
828
+ ----------
829
+ packages : Dict[str, str], default: {}
830
+ Packages to use for this step. The key is the name of the package
831
+ and the value is the version to use.
832
+ python : str, optional, default: None
833
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
834
+ that the version used will correspond to the version of the Python interpreter used to start the run.
835
+ """
836
+ ...
837
+
838
+ @typing.overload
839
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
840
+ ...
841
+
842
+ @typing.overload
843
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
844
+ ...
845
+
846
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
847
+ """
848
+ Specifies the PyPI packages for the step.
849
+
850
+ Information in this decorator will augment any
851
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
852
+ you can use `@pypi_base` to set packages required by all
853
+ steps and use `@pypi` to specify step-specific overrides.
1179
854
 
1180
855
 
1181
856
  Parameters
1182
857
  ----------
1183
- event : Union[str, Dict[str, Any]], optional, default None
1184
- Event dependency for this flow.
1185
- events : List[Union[str, Dict[str, Any]]], default []
1186
- Events dependency for this flow.
1187
- options : Dict[str, Any], default {}
1188
- Backend-specific configuration for tuning eventing behavior.
858
+ packages : Dict[str, str], default: {}
859
+ Packages to use for this step. The key is the name of the package
860
+ and the value is the version to use.
861
+ python : str, optional, default: None
862
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
863
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1189
864
  """
1190
865
  ...
1191
866
 
1192
867
  @typing.overload
1193
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
868
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
869
+ """
870
+ Internal decorator to support Fast bakery
871
+ """
1194
872
  ...
1195
873
 
1196
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
874
+ @typing.overload
875
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
876
+ ...
877
+
878
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1197
879
  """
1198
- Specifies the event(s) that this flow depends on.
880
+ Internal decorator to support Fast bakery
881
+ """
882
+ ...
883
+
884
+ @typing.overload
885
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
886
+ """
887
+ Specifies the resources needed when executing this step.
1199
888
 
889
+ Use `@resources` to specify the resource requirements
890
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
891
+
892
+ You can choose the compute layer on the command line by executing e.g.
1200
893
  ```
1201
- @trigger(event='foo')
894
+ python myflow.py run --with batch
1202
895
  ```
1203
896
  or
1204
897
  ```
1205
- @trigger(events=['foo', 'bar'])
898
+ python myflow.py run --with kubernetes
1206
899
  ```
900
+ which executes the flow on the desired system using the
901
+ requirements specified in `@resources`.
1207
902
 
1208
- Additionally, you can specify the parameter mappings
1209
- to map event payload to Metaflow parameters for the flow.
903
+
904
+ Parameters
905
+ ----------
906
+ cpu : int, default 1
907
+ Number of CPUs required for this step.
908
+ gpu : int, optional, default None
909
+ Number of GPUs required for this step.
910
+ disk : int, optional, default None
911
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
912
+ memory : int, default 4096
913
+ Memory size (in MB) required for this step.
914
+ shared_memory : int, optional, default None
915
+ The value for the size (in MiB) of the /dev/shm volume for this step.
916
+ This parameter maps to the `--shm-size` option in Docker.
917
+ """
918
+ ...
919
+
920
+ @typing.overload
921
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
922
+ ...
923
+
924
+ @typing.overload
925
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
926
+ ...
927
+
928
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
929
+ """
930
+ Specifies the resources needed when executing this step.
931
+
932
+ Use `@resources` to specify the resource requirements
933
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
934
+
935
+ You can choose the compute layer on the command line by executing e.g.
1210
936
  ```
1211
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
937
+ python myflow.py run --with batch
1212
938
  ```
1213
939
  or
1214
940
  ```
1215
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1216
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
941
+ python myflow.py run --with kubernetes
1217
942
  ```
943
+ which executes the flow on the desired system using the
944
+ requirements specified in `@resources`.
945
+
946
+
947
+ Parameters
948
+ ----------
949
+ cpu : int, default 1
950
+ Number of CPUs required for this step.
951
+ gpu : int, optional, default None
952
+ Number of GPUs required for this step.
953
+ disk : int, optional, default None
954
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
955
+ memory : int, default 4096
956
+ Memory size (in MB) required for this step.
957
+ shared_memory : int, optional, default None
958
+ The value for the size (in MiB) of the /dev/shm volume for this step.
959
+ This parameter maps to the `--shm-size` option in Docker.
960
+ """
961
+ ...
962
+
963
+ @typing.overload
964
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
965
+ """
966
+ Specifies the number of times the task corresponding
967
+ to a step needs to be retried.
1218
968
 
1219
- 'parameters' can also be a list of strings and tuples like so:
1220
- ```
1221
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1222
- ```
1223
- This is equivalent to:
1224
- ```
1225
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1226
- ```
969
+ This decorator is useful for handling transient errors, such as networking issues.
970
+ If your task contains operations that can't be retried safely, e.g. database updates,
971
+ it is advisable to annotate it with `@retry(times=0)`.
972
+
973
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
974
+ decorator will execute a no-op task after all retries have been exhausted,
975
+ ensuring that the flow execution can continue.
1227
976
 
1228
977
 
1229
978
  Parameters
1230
979
  ----------
1231
- event : Union[str, Dict[str, Any]], optional, default None
1232
- Event dependency for this flow.
1233
- events : List[Union[str, Dict[str, Any]]], default []
1234
- Events dependency for this flow.
1235
- options : Dict[str, Any], default {}
1236
- Backend-specific configuration for tuning eventing behavior.
980
+ times : int, default 3
981
+ Number of times to retry this task.
982
+ minutes_between_retries : int, default 2
983
+ Number of minutes between retries.
1237
984
  """
1238
985
  ...
1239
986
 
1240
987
  @typing.overload
1241
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
988
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
989
+ ...
990
+
991
+ @typing.overload
992
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
993
+ ...
994
+
995
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1242
996
  """
1243
- Specifies the times when the flow should be run when running on a
1244
- production scheduler.
997
+ Specifies the number of times the task corresponding
998
+ to a step needs to be retried.
999
+
1000
+ This decorator is useful for handling transient errors, such as networking issues.
1001
+ If your task contains operations that can't be retried safely, e.g. database updates,
1002
+ it is advisable to annotate it with `@retry(times=0)`.
1003
+
1004
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1005
+ decorator will execute a no-op task after all retries have been exhausted,
1006
+ ensuring that the flow execution can continue.
1245
1007
 
1246
1008
 
1247
1009
  Parameters
1248
1010
  ----------
1249
- hourly : bool, default False
1250
- Run the workflow hourly.
1251
- daily : bool, default True
1252
- Run the workflow daily.
1253
- weekly : bool, default False
1254
- Run the workflow weekly.
1255
- cron : str, optional, default None
1256
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1257
- specified by this expression.
1258
- timezone : str, optional, default None
1259
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1260
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1011
+ times : int, default 3
1012
+ Number of times to retry this task.
1013
+ minutes_between_retries : int, default 2
1014
+ Number of minutes between retries.
1261
1015
  """
1262
1016
  ...
1263
1017
 
1264
- @typing.overload
1265
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1266
- ...
1267
-
1268
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1018
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1269
1019
  """
1270
- Specifies the times when the flow should be run when running on a
1271
- production scheduler.
1020
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1021
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1272
1022
 
1273
1023
 
1274
1024
  Parameters
1275
1025
  ----------
1276
- hourly : bool, default False
1277
- Run the workflow hourly.
1278
- daily : bool, default True
1279
- Run the workflow daily.
1280
- weekly : bool, default False
1281
- Run the workflow weekly.
1282
- cron : str, optional, default None
1283
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1284
- specified by this expression.
1285
- timezone : str, optional, default None
1286
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1287
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1026
+ timeout : int
1027
+ Time, in seconds before the task times out and fails. (Default: 3600)
1028
+ poke_interval : int
1029
+ Time in seconds that the job should wait in between each try. (Default: 60)
1030
+ mode : str
1031
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1032
+ exponential_backoff : bool
1033
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1034
+ pool : str
1035
+ the slot pool this task should run in,
1036
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1037
+ soft_fail : bool
1038
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1039
+ name : str
1040
+ Name of the sensor on Airflow
1041
+ description : str
1042
+ Description of sensor in the Airflow UI
1043
+ external_dag_id : str
1044
+ The dag_id that contains the task you want to wait for.
1045
+ external_task_ids : List[str]
1046
+ The list of task_ids that you want to wait for.
1047
+ If None (default value) the sensor waits for the DAG. (Default: None)
1048
+ allowed_states : List[str]
1049
+ Iterable of allowed states, (Default: ['success'])
1050
+ failed_states : List[str]
1051
+ Iterable of failed or dis-allowed states. (Default: None)
1052
+ execution_delta : datetime.timedelta
1053
+ time difference with the previous execution to look at,
1054
+ the default is the same logical date as the current task or DAG. (Default: None)
1055
+ check_existence: bool
1056
+ Set to True to check if the external task exists or check if
1057
+ the DAG to wait for exists. (Default: True)
1288
1058
  """
1289
1059
  ...
1290
1060
 
@@ -1402,46 +1172,155 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1402
1172
  """
1403
1173
  ...
1404
1174
 
1405
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1175
+ @typing.overload
1176
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1406
1177
  """
1407
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1408
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1409
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1410
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1411
- starts only after all sensors finish.
1178
+ Specifies the flow(s) that this flow depends on.
1179
+
1180
+ ```
1181
+ @trigger_on_finish(flow='FooFlow')
1182
+ ```
1183
+ or
1184
+ ```
1185
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1186
+ ```
1187
+ This decorator respects the @project decorator and triggers the flow
1188
+ when upstream runs within the same namespace complete successfully
1189
+
1190
+ Additionally, you can specify project aware upstream flow dependencies
1191
+ by specifying the fully qualified project_flow_name.
1192
+ ```
1193
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1194
+ ```
1195
+ or
1196
+ ```
1197
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1198
+ ```
1199
+
1200
+ You can also specify just the project or project branch (other values will be
1201
+ inferred from the current project or project branch):
1202
+ ```
1203
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1204
+ ```
1205
+
1206
+ Note that `branch` is typically one of:
1207
+ - `prod`
1208
+ - `user.bob`
1209
+ - `test.my_experiment`
1210
+ - `prod.staging`
1211
+
1212
+
1213
+ Parameters
1214
+ ----------
1215
+ flow : Union[str, Dict[str, str]], optional, default None
1216
+ Upstream flow dependency for this flow.
1217
+ flows : List[Union[str, Dict[str, str]]], default []
1218
+ Upstream flow dependencies for this flow.
1219
+ options : Dict[str, Any], default {}
1220
+ Backend-specific configuration for tuning eventing behavior.
1221
+ """
1222
+ ...
1223
+
1224
+ @typing.overload
1225
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1226
+ ...
1227
+
1228
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1229
+ """
1230
+ Specifies the flow(s) that this flow depends on.
1231
+
1232
+ ```
1233
+ @trigger_on_finish(flow='FooFlow')
1234
+ ```
1235
+ or
1236
+ ```
1237
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1238
+ ```
1239
+ This decorator respects the @project decorator and triggers the flow
1240
+ when upstream runs within the same namespace complete successfully
1241
+
1242
+ Additionally, you can specify project aware upstream flow dependencies
1243
+ by specifying the fully qualified project_flow_name.
1244
+ ```
1245
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1246
+ ```
1247
+ or
1248
+ ```
1249
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1250
+ ```
1251
+
1252
+ You can also specify just the project or project branch (other values will be
1253
+ inferred from the current project or project branch):
1254
+ ```
1255
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1256
+ ```
1257
+
1258
+ Note that `branch` is typically one of:
1259
+ - `prod`
1260
+ - `user.bob`
1261
+ - `test.my_experiment`
1262
+ - `prod.staging`
1263
+
1264
+
1265
+ Parameters
1266
+ ----------
1267
+ flow : Union[str, Dict[str, str]], optional, default None
1268
+ Upstream flow dependency for this flow.
1269
+ flows : List[Union[str, Dict[str, str]]], default []
1270
+ Upstream flow dependencies for this flow.
1271
+ options : Dict[str, Any], default {}
1272
+ Backend-specific configuration for tuning eventing behavior.
1273
+ """
1274
+ ...
1275
+
1276
+ @typing.overload
1277
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1278
+ """
1279
+ Specifies the times when the flow should be run when running on a
1280
+ production scheduler.
1281
+
1282
+
1283
+ Parameters
1284
+ ----------
1285
+ hourly : bool, default False
1286
+ Run the workflow hourly.
1287
+ daily : bool, default True
1288
+ Run the workflow daily.
1289
+ weekly : bool, default False
1290
+ Run the workflow weekly.
1291
+ cron : str, optional, default None
1292
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1293
+ specified by this expression.
1294
+ timezone : str, optional, default None
1295
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1296
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1297
+ """
1298
+ ...
1299
+
1300
+ @typing.overload
1301
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1302
+ ...
1303
+
1304
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1305
+ """
1306
+ Specifies the times when the flow should be run when running on a
1307
+ production scheduler.
1412
1308
 
1413
1309
 
1414
1310
  Parameters
1415
1311
  ----------
1416
- timeout : int
1417
- Time, in seconds before the task times out and fails. (Default: 3600)
1418
- poke_interval : int
1419
- Time in seconds that the job should wait in between each try. (Default: 60)
1420
- mode : str
1421
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1422
- exponential_backoff : bool
1423
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1424
- pool : str
1425
- the slot pool this task should run in,
1426
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1427
- soft_fail : bool
1428
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1429
- name : str
1430
- Name of the sensor on Airflow
1431
- description : str
1432
- Description of sensor in the Airflow UI
1433
- bucket_key : Union[str, List[str]]
1434
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1435
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1436
- bucket_name : str
1437
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1438
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1439
- wildcard_match : bool
1440
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1441
- aws_conn_id : str
1442
- a reference to the s3 connection on Airflow. (Default: None)
1443
- verify : bool
1444
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1312
+ hourly : bool, default False
1313
+ Run the workflow hourly.
1314
+ daily : bool, default True
1315
+ Run the workflow daily.
1316
+ weekly : bool, default False
1317
+ Run the workflow weekly.
1318
+ cron : str, optional, default None
1319
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1320
+ specified by this expression.
1321
+ timezone : str, optional, default None
1322
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1323
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1445
1324
  """
1446
1325
  ...
1447
1326
 
@@ -1487,105 +1366,226 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
1487
1366
  ...
1488
1367
 
1489
1368
  @typing.overload
1490
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1369
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1491
1370
  """
1492
- Specifies the flow(s) that this flow depends on.
1371
+ Specifies the event(s) that this flow depends on.
1493
1372
 
1494
1373
  ```
1495
- @trigger_on_finish(flow='FooFlow')
1374
+ @trigger(event='foo')
1496
1375
  ```
1497
1376
  or
1498
1377
  ```
1499
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1378
+ @trigger(events=['foo', 'bar'])
1500
1379
  ```
1501
- This decorator respects the @project decorator and triggers the flow
1502
- when upstream runs within the same namespace complete successfully
1503
1380
 
1504
- Additionally, you can specify project aware upstream flow dependencies
1505
- by specifying the fully qualified project_flow_name.
1381
+ Additionally, you can specify the parameter mappings
1382
+ to map event payload to Metaflow parameters for the flow.
1506
1383
  ```
1507
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1384
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1508
1385
  ```
1509
1386
  or
1510
1387
  ```
1511
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1388
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1389
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1512
1390
  ```
1513
1391
 
1514
- You can also specify just the project or project branch (other values will be
1515
- inferred from the current project or project branch):
1392
+ 'parameters' can also be a list of strings and tuples like so:
1516
1393
  ```
1517
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1394
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1395
+ ```
1396
+ This is equivalent to:
1397
+ ```
1398
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1518
1399
  ```
1519
-
1520
- Note that `branch` is typically one of:
1521
- - `prod`
1522
- - `user.bob`
1523
- - `test.my_experiment`
1524
- - `prod.staging`
1525
1400
 
1526
1401
 
1527
1402
  Parameters
1528
1403
  ----------
1529
- flow : Union[str, Dict[str, str]], optional, default None
1530
- Upstream flow dependency for this flow.
1531
- flows : List[Union[str, Dict[str, str]]], default []
1532
- Upstream flow dependencies for this flow.
1404
+ event : Union[str, Dict[str, Any]], optional, default None
1405
+ Event dependency for this flow.
1406
+ events : List[Union[str, Dict[str, Any]]], default []
1407
+ Events dependency for this flow.
1533
1408
  options : Dict[str, Any], default {}
1534
1409
  Backend-specific configuration for tuning eventing behavior.
1535
1410
  """
1536
1411
  ...
1537
1412
 
1538
1413
  @typing.overload
1539
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1414
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1540
1415
  ...
1541
1416
 
1542
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1417
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1543
1418
  """
1544
- Specifies the flow(s) that this flow depends on.
1419
+ Specifies the event(s) that this flow depends on.
1545
1420
 
1546
1421
  ```
1547
- @trigger_on_finish(flow='FooFlow')
1422
+ @trigger(event='foo')
1548
1423
  ```
1549
1424
  or
1550
1425
  ```
1551
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1426
+ @trigger(events=['foo', 'bar'])
1552
1427
  ```
1553
- This decorator respects the @project decorator and triggers the flow
1554
- when upstream runs within the same namespace complete successfully
1555
1428
 
1556
- Additionally, you can specify project aware upstream flow dependencies
1557
- by specifying the fully qualified project_flow_name.
1429
+ Additionally, you can specify the parameter mappings
1430
+ to map event payload to Metaflow parameters for the flow.
1558
1431
  ```
1559
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1432
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1560
1433
  ```
1561
1434
  or
1562
1435
  ```
1563
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1436
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1437
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1564
1438
  ```
1565
1439
 
1566
- You can also specify just the project or project branch (other values will be
1567
- inferred from the current project or project branch):
1440
+ 'parameters' can also be a list of strings and tuples like so:
1568
1441
  ```
1569
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1442
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1443
+ ```
1444
+ This is equivalent to:
1445
+ ```
1446
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1570
1447
  ```
1571
-
1572
- Note that `branch` is typically one of:
1573
- - `prod`
1574
- - `user.bob`
1575
- - `test.my_experiment`
1576
- - `prod.staging`
1577
1448
 
1578
1449
 
1579
1450
  Parameters
1580
1451
  ----------
1581
- flow : Union[str, Dict[str, str]], optional, default None
1582
- Upstream flow dependency for this flow.
1583
- flows : List[Union[str, Dict[str, str]]], default []
1584
- Upstream flow dependencies for this flow.
1452
+ event : Union[str, Dict[str, Any]], optional, default None
1453
+ Event dependency for this flow.
1454
+ events : List[Union[str, Dict[str, Any]]], default []
1455
+ Events dependency for this flow.
1585
1456
  options : Dict[str, Any], default {}
1586
1457
  Backend-specific configuration for tuning eventing behavior.
1587
1458
  """
1588
1459
  ...
1589
1460
 
1461
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1462
+ """
1463
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1464
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1465
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1466
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1467
+ starts only after all sensors finish.
1468
+
1469
+
1470
+ Parameters
1471
+ ----------
1472
+ timeout : int
1473
+ Time, in seconds before the task times out and fails. (Default: 3600)
1474
+ poke_interval : int
1475
+ Time in seconds that the job should wait in between each try. (Default: 60)
1476
+ mode : str
1477
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1478
+ exponential_backoff : bool
1479
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1480
+ pool : str
1481
+ the slot pool this task should run in,
1482
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1483
+ soft_fail : bool
1484
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1485
+ name : str
1486
+ Name of the sensor on Airflow
1487
+ description : str
1488
+ Description of sensor in the Airflow UI
1489
+ bucket_key : Union[str, List[str]]
1490
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1491
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1492
+ bucket_name : str
1493
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1494
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1495
+ wildcard_match : bool
1496
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1497
+ aws_conn_id : str
1498
+ a reference to the s3 connection on Airflow. (Default: None)
1499
+ verify : bool
1500
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1501
+ """
1502
+ ...
1503
+
1504
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1505
+ """
1506
+ Specifies what flows belong to the same project.
1507
+
1508
+ A project-specific namespace is created for all flows that
1509
+ use the same `@project(name)`.
1510
+
1511
+
1512
+ Parameters
1513
+ ----------
1514
+ name : str
1515
+ Project name. Make sure that the name is unique amongst all
1516
+ projects that use the same production scheduler. The name may
1517
+ contain only lowercase alphanumeric characters and underscores.
1518
+
1519
+ branch : Optional[str], default None
1520
+ The branch to use. If not specified, the branch is set to
1521
+ `user.<username>` unless `production` is set to `True`. This can
1522
+ also be set on the command line using `--branch` as a top-level option.
1523
+ It is an error to specify `branch` in the decorator and on the command line.
1524
+
1525
+ production : bool, default False
1526
+ Whether or not the branch is the production branch. This can also be set on the
1527
+ command line using `--production` as a top-level option. It is an error to specify
1528
+ `production` in the decorator and on the command line.
1529
+ The project branch name will be:
1530
+ - if `branch` is specified:
1531
+ - if `production` is True: `prod.<branch>`
1532
+ - if `production` is False: `test.<branch>`
1533
+ - if `branch` is not specified:
1534
+ - if `production` is True: `prod`
1535
+ - if `production` is False: `user.<username>`
1536
+ """
1537
+ ...
1538
+
1539
+ @typing.overload
1540
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1541
+ """
1542
+ Specifies the Conda environment for all steps of the flow.
1543
+
1544
+ Use `@conda_base` to set common libraries required by all
1545
+ steps and use `@conda` to specify step-specific additions.
1546
+
1547
+
1548
+ Parameters
1549
+ ----------
1550
+ packages : Dict[str, str], default {}
1551
+ Packages to use for this flow. The key is the name of the package
1552
+ and the value is the version to use.
1553
+ libraries : Dict[str, str], default {}
1554
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1555
+ python : str, optional, default None
1556
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1557
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1558
+ disabled : bool, default False
1559
+ If set to True, disables Conda.
1560
+ """
1561
+ ...
1562
+
1563
+ @typing.overload
1564
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1565
+ ...
1566
+
1567
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1568
+ """
1569
+ Specifies the Conda environment for all steps of the flow.
1570
+
1571
+ Use `@conda_base` to set common libraries required by all
1572
+ steps and use `@conda` to specify step-specific additions.
1573
+
1574
+
1575
+ Parameters
1576
+ ----------
1577
+ packages : Dict[str, str], default {}
1578
+ Packages to use for this flow. The key is the name of the package
1579
+ and the value is the version to use.
1580
+ libraries : Dict[str, str], default {}
1581
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1582
+ python : str, optional, default None
1583
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1584
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1585
+ disabled : bool, default False
1586
+ If set to True, disables Conda.
1587
+ """
1588
+ ...
1589
+
1590
1590
  pkg_name: str
1591
1591