ob-metaflow-stubs 6.0.3.175rc1__py2.py3-none-any.whl → 6.0.3.176rc2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (218) hide show
  1. metaflow-stubs/__init__.pyi +718 -712
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +1 -1
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +90 -90
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +1 -1
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +1 -1
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +1 -1
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +60 -0
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +60 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +73 -0
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  85. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  86. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  87. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
  88. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  89. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
  90. metaflow-stubs/multicore_utils.pyi +1 -1
  91. metaflow-stubs/ob_internal.pyi +1 -1
  92. metaflow-stubs/parameters.pyi +3 -3
  93. metaflow-stubs/plugins/__init__.pyi +11 -11
  94. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  95. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  96. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  97. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  98. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  99. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  100. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  101. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  102. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  103. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  104. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  105. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +1 -1
  106. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  107. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  108. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  109. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  110. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  111. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  112. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  113. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  114. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  115. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  116. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  117. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  118. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  119. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  120. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  121. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  122. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  123. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  124. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  125. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  126. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  127. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  128. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  129. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  130. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  131. metaflow-stubs/plugins/cards/__init__.pyi +5 -5
  132. metaflow-stubs/plugins/cards/card_client.pyi +1 -1
  133. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  134. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  135. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  136. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  137. metaflow-stubs/plugins/cards/card_modules/basic.pyi +1 -1
  138. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  139. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  140. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  141. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  142. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  143. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  144. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  145. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  146. metaflow-stubs/plugins/catch_decorator.pyi +1 -1
  147. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  148. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  149. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  150. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  151. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  152. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  153. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  154. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  155. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  156. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  157. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  158. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  159. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  160. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  161. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  162. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  163. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  164. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  165. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  166. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  167. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  168. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  169. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  170. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  171. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  172. metaflow-stubs/plugins/ollama/__init__.pyi +12 -3
  173. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  174. metaflow-stubs/plugins/perimeters.pyi +1 -1
  175. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  176. metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
  177. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  178. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  179. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  180. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  181. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  182. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  183. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  184. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  185. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  186. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  187. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  188. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  189. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  190. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
  191. metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
  192. metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
  193. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  194. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  195. metaflow-stubs/profilers/__init__.pyi +1 -1
  196. metaflow-stubs/pylint_wrapper.pyi +1 -1
  197. metaflow-stubs/runner/__init__.pyi +1 -1
  198. metaflow-stubs/runner/deployer.pyi +6 -6
  199. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  200. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  201. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  202. metaflow-stubs/runner/nbrun.pyi +1 -1
  203. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  204. metaflow-stubs/runner/utils.pyi +2 -2
  205. metaflow-stubs/system/__init__.pyi +1 -1
  206. metaflow-stubs/system/system_logger.pyi +1 -1
  207. metaflow-stubs/system/system_monitor.pyi +1 -1
  208. metaflow-stubs/tagging_util.pyi +1 -1
  209. metaflow-stubs/tuple_util.pyi +1 -1
  210. metaflow-stubs/user_configs/__init__.pyi +1 -1
  211. metaflow-stubs/user_configs/config_decorators.pyi +3 -3
  212. metaflow-stubs/user_configs/config_options.pyi +3 -3
  213. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  214. {ob_metaflow_stubs-6.0.3.175rc1.dist-info → ob_metaflow_stubs-6.0.3.176rc2.dist-info}/METADATA +1 -1
  215. ob_metaflow_stubs-6.0.3.176rc2.dist-info/RECORD +218 -0
  216. ob_metaflow_stubs-6.0.3.175rc1.dist-info/RECORD +0 -216
  217. {ob_metaflow_stubs-6.0.3.175rc1.dist-info → ob_metaflow_stubs-6.0.3.176rc2.dist-info}/WHEEL +0 -0
  218. {ob_metaflow_stubs-6.0.3.175rc1.dist-info → ob_metaflow_stubs-6.0.3.176rc2.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.15.14.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-05-31T01:09:57.775269 #
4
+ # Generated on 2025-06-03T03:11:47.762585 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -35,17 +35,17 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
- from . import tuple_util as tuple_util
39
38
  from . import cards as cards
40
- from . import events as events
41
39
  from . import metaflow_git as metaflow_git
40
+ from . import tuple_util as tuple_util
41
+ from . import events as events
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
48
47
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
48
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
49
49
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
@@ -154,6 +154,133 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
154
154
  """
155
155
  ...
156
156
 
157
+ @typing.overload
158
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
159
+ """
160
+ Decorator prototype for all step decorators. This function gets specialized
161
+ and imported for all decorators types by _import_plugin_decorators().
162
+ """
163
+ ...
164
+
165
+ @typing.overload
166
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
167
+ ...
168
+
169
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
170
+ """
171
+ Decorator prototype for all step decorators. This function gets specialized
172
+ and imported for all decorators types by _import_plugin_decorators().
173
+ """
174
+ ...
175
+
176
+ @typing.overload
177
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
178
+ """
179
+ Creates a human-readable report, a Metaflow Card, after this step completes.
180
+
181
+ Note that you may add multiple `@card` decorators in a step with different parameters.
182
+
183
+
184
+ Parameters
185
+ ----------
186
+ type : str, default 'default'
187
+ Card type.
188
+ id : str, optional, default None
189
+ If multiple cards are present, use this id to identify this card.
190
+ options : Dict[str, Any], default {}
191
+ Options passed to the card. The contents depend on the card type.
192
+ timeout : int, default 45
193
+ Interrupt reporting if it takes more than this many seconds.
194
+ """
195
+ ...
196
+
197
+ @typing.overload
198
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
199
+ ...
200
+
201
+ @typing.overload
202
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
203
+ ...
204
+
205
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
206
+ """
207
+ Creates a human-readable report, a Metaflow Card, after this step completes.
208
+
209
+ Note that you may add multiple `@card` decorators in a step with different parameters.
210
+
211
+
212
+ Parameters
213
+ ----------
214
+ type : str, default 'default'
215
+ Card type.
216
+ id : str, optional, default None
217
+ If multiple cards are present, use this id to identify this card.
218
+ options : Dict[str, Any], default {}
219
+ Options passed to the card. The contents depend on the card type.
220
+ timeout : int, default 45
221
+ Interrupt reporting if it takes more than this many seconds.
222
+ """
223
+ ...
224
+
225
+ @typing.overload
226
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
227
+ """
228
+ Specifies a timeout for your step.
229
+
230
+ This decorator is useful if this step may hang indefinitely.
231
+
232
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
233
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
234
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
235
+
236
+ Note that all the values specified in parameters are added together so if you specify
237
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
238
+
239
+
240
+ Parameters
241
+ ----------
242
+ seconds : int, default 0
243
+ Number of seconds to wait prior to timing out.
244
+ minutes : int, default 0
245
+ Number of minutes to wait prior to timing out.
246
+ hours : int, default 0
247
+ Number of hours to wait prior to timing out.
248
+ """
249
+ ...
250
+
251
+ @typing.overload
252
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
253
+ ...
254
+
255
+ @typing.overload
256
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
257
+ ...
258
+
259
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
260
+ """
261
+ Specifies a timeout for your step.
262
+
263
+ This decorator is useful if this step may hang indefinitely.
264
+
265
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
266
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
267
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
268
+
269
+ Note that all the values specified in parameters are added together so if you specify
270
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
271
+
272
+
273
+ Parameters
274
+ ----------
275
+ seconds : int, default 0
276
+ Number of seconds to wait prior to timing out.
277
+ minutes : int, default 0
278
+ Number of minutes to wait prior to timing out.
279
+ hours : int, default 0
280
+ Number of hours to wait prior to timing out.
281
+ """
282
+ ...
283
+
157
284
  @typing.overload
158
285
  def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
159
286
  """
@@ -234,297 +361,138 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
234
361
  ...
235
362
 
236
363
  @typing.overload
237
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
364
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
238
365
  """
239
- Specifies that the step will success under all circumstances.
366
+ Specifies the Conda environment for the step.
240
367
 
241
- The decorator will create an optional artifact, specified by `var`, which
242
- contains the exception raised. You can use it to detect the presence
243
- of errors, indicating that all happy-path artifacts produced by the step
244
- are missing.
368
+ Information in this decorator will augment any
369
+ attributes set in the `@conda_base` flow-level decorator. Hence,
370
+ you can use `@conda_base` to set packages required by all
371
+ steps and use `@conda` to specify step-specific overrides.
245
372
 
246
373
 
247
374
  Parameters
248
375
  ----------
249
- var : str, optional, default None
250
- Name of the artifact in which to store the caught exception.
251
- If not specified, the exception is not stored.
252
- print_exception : bool, default True
253
- Determines whether or not the exception is printed to
254
- stdout when caught.
376
+ packages : Dict[str, str], default {}
377
+ Packages to use for this step. The key is the name of the package
378
+ and the value is the version to use.
379
+ libraries : Dict[str, str], default {}
380
+ Supported for backward compatibility. When used with packages, packages will take precedence.
381
+ python : str, optional, default None
382
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
383
+ that the version used will correspond to the version of the Python interpreter used to start the run.
384
+ disabled : bool, default False
385
+ If set to True, disables @conda.
255
386
  """
256
387
  ...
257
388
 
258
389
  @typing.overload
259
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
390
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
260
391
  ...
261
392
 
262
393
  @typing.overload
263
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
394
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
264
395
  ...
265
396
 
266
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
397
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
267
398
  """
268
- Specifies that the step will success under all circumstances.
399
+ Specifies the Conda environment for the step.
269
400
 
270
- The decorator will create an optional artifact, specified by `var`, which
271
- contains the exception raised. You can use it to detect the presence
272
- of errors, indicating that all happy-path artifacts produced by the step
273
- are missing.
401
+ Information in this decorator will augment any
402
+ attributes set in the `@conda_base` flow-level decorator. Hence,
403
+ you can use `@conda_base` to set packages required by all
404
+ steps and use `@conda` to specify step-specific overrides.
274
405
 
275
406
 
276
407
  Parameters
277
408
  ----------
278
- var : str, optional, default None
279
- Name of the artifact in which to store the caught exception.
280
- If not specified, the exception is not stored.
281
- print_exception : bool, default True
282
- Determines whether or not the exception is printed to
283
- stdout when caught.
409
+ packages : Dict[str, str], default {}
410
+ Packages to use for this step. The key is the name of the package
411
+ and the value is the version to use.
412
+ libraries : Dict[str, str], default {}
413
+ Supported for backward compatibility. When used with packages, packages will take precedence.
414
+ python : str, optional, default None
415
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
416
+ that the version used will correspond to the version of the Python interpreter used to start the run.
417
+ disabled : bool, default False
418
+ If set to True, disables @conda.
284
419
  """
285
420
  ...
286
421
 
287
- @typing.overload
288
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
422
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
289
423
  """
290
- Enables loading / saving of models within a step.
291
-
424
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
292
425
 
293
426
 
294
427
  Parameters
295
428
  ----------
296
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
297
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
298
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
299
- - `current.checkpoint`
300
- - `current.model`
301
- - `current.huggingface_hub`
429
+ temp_dir_root : str, optional
430
+ The root directory that will hold the temporary directory where objects will be downloaded.
302
431
 
303
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
304
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
305
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
432
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
433
+ The list of repos (models/datasets) to load.
306
434
 
307
- temp_dir_root : str, default: None
308
- The root directory under which `current.model.loaded` will store loaded models
435
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
436
+
437
+ - If repo (model/dataset) is not found in the datastore:
438
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
439
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
440
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
441
+
442
+ - If repo is found in the datastore:
443
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
309
444
  """
310
445
  ...
311
446
 
312
447
  @typing.overload
313
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
448
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
449
+ """
450
+ Internal decorator to support Fast bakery
451
+ """
314
452
  ...
315
453
 
316
454
  @typing.overload
317
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
455
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
318
456
  ...
319
457
 
320
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
458
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
321
459
  """
322
- Enables loading / saving of models within a step.
323
-
324
-
325
-
326
- Parameters
327
- ----------
328
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
329
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
330
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
331
- - `current.checkpoint`
332
- - `current.model`
333
- - `current.huggingface_hub`
334
-
335
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
336
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
337
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
338
-
339
- temp_dir_root : str, default: None
340
- The root directory under which `current.model.loaded` will store loaded models
460
+ Internal decorator to support Fast bakery
341
461
  """
342
462
  ...
343
463
 
344
- def ollama(*, models: list, backend: str, force_pull: bool, skip_push_check: bool, debug: bool) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
464
+ @typing.overload
465
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
345
466
  """
346
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
347
-
348
- User code call
349
- --------------
350
- @ollama(
351
- models=[...],
352
- ...
353
- )
354
-
355
- Valid backend options
356
- ---------------------
357
- - 'local': Run as a separate process on the local task machine.
358
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
359
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
360
-
361
- Valid model options
362
- -------------------
363
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
364
-
365
-
366
- Parameters
367
- ----------
368
- models: list[str]
369
- List of Ollama containers running models in sidecars.
370
- backend: str
371
- Determines where and how to run the Ollama process.
372
- force_pull: bool
373
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
374
- skip_push_check: bool
375
- Whether to skip the check that populates/overwrites remote cache on terminating an ollama model.
376
- debug: bool
377
- Whether to turn on verbose debugging logs.
378
- """
379
- ...
380
-
381
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
382
- """
383
- Specifies that this step is used to deploy an instance of the app.
384
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
467
+ Specifies secrets to be retrieved and injected as environment variables prior to
468
+ the execution of a step.
385
469
 
386
470
 
387
471
  Parameters
388
472
  ----------
389
- app_port : int
390
- Number of GPUs to use.
391
- app_name : str
392
- Name of the app to deploy.
393
- """
394
- ...
395
-
396
- @typing.overload
397
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
398
- """
399
- Decorator prototype for all step decorators. This function gets specialized
400
- and imported for all decorators types by _import_plugin_decorators().
401
- """
402
- ...
403
-
404
- @typing.overload
405
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
406
- ...
407
-
408
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
409
- """
410
- Decorator prototype for all step decorators. This function gets specialized
411
- and imported for all decorators types by _import_plugin_decorators().
473
+ sources : List[Union[str, Dict[str, Any]]], default: []
474
+ List of secret specs, defining how the secrets are to be retrieved
412
475
  """
413
476
  ...
414
477
 
415
478
  @typing.overload
416
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
417
- """
418
- Internal decorator to support Fast bakery
419
- """
479
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
420
480
  ...
421
481
 
422
482
  @typing.overload
423
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
424
- ...
425
-
426
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
427
- """
428
- Internal decorator to support Fast bakery
429
- """
430
- ...
431
-
432
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
433
- """
434
- Specifies that this step should execute on DGX cloud.
435
-
436
-
437
- Parameters
438
- ----------
439
- gpu : int
440
- Number of GPUs to use.
441
- gpu_type : str
442
- Type of Nvidia GPU to use.
443
- """
483
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
444
484
  ...
445
485
 
446
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
486
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
447
487
  """
448
- Specifies that this step should execute on Kubernetes.
488
+ Specifies secrets to be retrieved and injected as environment variables prior to
489
+ the execution of a step.
449
490
 
450
491
 
451
492
  Parameters
452
493
  ----------
453
- cpu : int, default 1
454
- Number of CPUs required for this step. If `@resources` is
455
- also present, the maximum value from all decorators is used.
456
- memory : int, default 4096
457
- Memory size (in MB) required for this step. If
458
- `@resources` is also present, the maximum value from all decorators is
459
- used.
460
- disk : int, default 10240
461
- Disk size (in MB) required for this step. If
462
- `@resources` is also present, the maximum value from all decorators is
463
- used.
464
- image : str, optional, default None
465
- Docker image to use when launching on Kubernetes. If not specified, and
466
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
467
- not, a default Docker image mapping to the current version of Python is used.
468
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
469
- If given, the imagePullPolicy to be applied to the Docker image of the step.
470
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
471
- Kubernetes service account to use when launching pod in Kubernetes.
472
- secrets : List[str], optional, default None
473
- Kubernetes secrets to use when launching pod in Kubernetes. These
474
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
475
- in Metaflow configuration.
476
- node_selector: Union[Dict[str,str], str], optional, default None
477
- Kubernetes node selector(s) to apply to the pod running the task.
478
- Can be passed in as a comma separated string of values e.g.
479
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
480
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
481
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
482
- Kubernetes namespace to use when launching pod in Kubernetes.
483
- gpu : int, optional, default None
484
- Number of GPUs required for this step. A value of zero implies that
485
- the scheduled node should not have GPUs.
486
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
487
- The vendor of the GPUs to be used for this step.
488
- tolerations : List[str], default []
489
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
490
- Kubernetes tolerations to use when launching pod in Kubernetes.
491
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
492
- Kubernetes labels to use when launching pod in Kubernetes.
493
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
494
- Kubernetes annotations to use when launching pod in Kubernetes.
495
- use_tmpfs : bool, default False
496
- This enables an explicit tmpfs mount for this step.
497
- tmpfs_tempdir : bool, default True
498
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
499
- tmpfs_size : int, optional, default: None
500
- The value for the size (in MiB) of the tmpfs mount for this step.
501
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
502
- memory allocated for this step.
503
- tmpfs_path : str, optional, default /metaflow_temp
504
- Path to tmpfs mount for this step.
505
- persistent_volume_claims : Dict[str, str], optional, default None
506
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
507
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
508
- shared_memory: int, optional
509
- Shared memory size (in MiB) required for this step
510
- port: int, optional
511
- Port number to specify in the Kubernetes job object
512
- compute_pool : str, optional, default None
513
- Compute pool to be used for for this step.
514
- If not specified, any accessible compute pool within the perimeter is used.
515
- hostname_resolution_timeout: int, default 10 * 60
516
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
517
- Only applicable when @parallel is used.
518
- qos: str, default: Burstable
519
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
520
-
521
- security_context: Dict[str, Any], optional, default None
522
- Container security context. Applies to the task container. Allows the following keys:
523
- - privileged: bool, optional, default None
524
- - allow_privilege_escalation: bool, optional, default None
525
- - run_as_user: int, optional, default None
526
- - run_as_group: int, optional, default None
527
- - run_as_non_root: bool, optional, default None
494
+ sources : List[Union[str, Dict[str, Any]]], default: []
495
+ List of secret specs, defining how the secrets are to be retrieved
528
496
  """
529
497
  ...
530
498
 
@@ -577,198 +545,75 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
577
545
  """
578
546
  ...
579
547
 
580
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
548
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
581
549
  """
582
- Decorator that helps cache, version and store models/datasets from huggingface hub.
550
+ Specifies that this step should execute on DGX cloud.
583
551
 
584
552
 
585
553
  Parameters
586
554
  ----------
587
- temp_dir_root : str, optional
588
- The root directory that will hold the temporary directory where objects will be downloaded.
589
-
590
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
591
- The list of repos (models/datasets) to load.
592
-
593
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
594
-
595
- - If repo (model/dataset) is not found in the datastore:
596
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
597
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
598
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
599
-
600
- - If repo is found in the datastore:
601
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
555
+ gpu : int
556
+ Number of GPUs to use.
557
+ gpu_type : str
558
+ Type of Nvidia GPU to use.
602
559
  """
603
560
  ...
604
561
 
605
562
  @typing.overload
606
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
563
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
607
564
  """
608
- Specifies the number of times the task corresponding
609
- to a step needs to be retried.
610
-
611
- This decorator is useful for handling transient errors, such as networking issues.
612
- If your task contains operations that can't be retried safely, e.g. database updates,
613
- it is advisable to annotate it with `@retry(times=0)`.
565
+ Specifies the PyPI packages for the step.
614
566
 
615
- This can be used in conjunction with the `@catch` decorator. The `@catch`
616
- decorator will execute a no-op task after all retries have been exhausted,
617
- ensuring that the flow execution can continue.
567
+ Information in this decorator will augment any
568
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
569
+ you can use `@pypi_base` to set packages required by all
570
+ steps and use `@pypi` to specify step-specific overrides.
618
571
 
619
572
 
620
573
  Parameters
621
574
  ----------
622
- times : int, default 3
623
- Number of times to retry this task.
624
- minutes_between_retries : int, default 2
625
- Number of minutes between retries.
575
+ packages : Dict[str, str], default: {}
576
+ Packages to use for this step. The key is the name of the package
577
+ and the value is the version to use.
578
+ python : str, optional, default: None
579
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
580
+ that the version used will correspond to the version of the Python interpreter used to start the run.
626
581
  """
627
582
  ...
628
583
 
629
584
  @typing.overload
630
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
585
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
631
586
  ...
632
587
 
633
588
  @typing.overload
634
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
589
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
635
590
  ...
636
591
 
637
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
592
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
638
593
  """
639
- Specifies the number of times the task corresponding
640
- to a step needs to be retried.
641
-
642
- This decorator is useful for handling transient errors, such as networking issues.
643
- If your task contains operations that can't be retried safely, e.g. database updates,
644
- it is advisable to annotate it with `@retry(times=0)`.
594
+ Specifies the PyPI packages for the step.
645
595
 
646
- This can be used in conjunction with the `@catch` decorator. The `@catch`
647
- decorator will execute a no-op task after all retries have been exhausted,
648
- ensuring that the flow execution can continue.
596
+ Information in this decorator will augment any
597
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
598
+ you can use `@pypi_base` to set packages required by all
599
+ steps and use `@pypi` to specify step-specific overrides.
649
600
 
650
601
 
651
602
  Parameters
652
603
  ----------
653
- times : int, default 3
654
- Number of times to retry this task.
655
- minutes_between_retries : int, default 2
656
- Number of minutes between retries.
604
+ packages : Dict[str, str], default: {}
605
+ Packages to use for this step. The key is the name of the package
606
+ and the value is the version to use.
607
+ python : str, optional, default: None
608
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
609
+ that the version used will correspond to the version of the Python interpreter used to start the run.
657
610
  """
658
611
  ...
659
612
 
660
613
  @typing.overload
661
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
614
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
662
615
  """
663
- Specifies a timeout for your step.
664
-
665
- This decorator is useful if this step may hang indefinitely.
666
-
667
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
668
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
669
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
670
-
671
- Note that all the values specified in parameters are added together so if you specify
672
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
673
-
674
-
675
- Parameters
676
- ----------
677
- seconds : int, default 0
678
- Number of seconds to wait prior to timing out.
679
- minutes : int, default 0
680
- Number of minutes to wait prior to timing out.
681
- hours : int, default 0
682
- Number of hours to wait prior to timing out.
683
- """
684
- ...
685
-
686
- @typing.overload
687
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
688
- ...
689
-
690
- @typing.overload
691
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
692
- ...
693
-
694
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
695
- """
696
- Specifies a timeout for your step.
697
-
698
- This decorator is useful if this step may hang indefinitely.
699
-
700
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
701
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
702
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
703
-
704
- Note that all the values specified in parameters are added together so if you specify
705
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
706
-
707
-
708
- Parameters
709
- ----------
710
- seconds : int, default 0
711
- Number of seconds to wait prior to timing out.
712
- minutes : int, default 0
713
- Number of minutes to wait prior to timing out.
714
- hours : int, default 0
715
- Number of hours to wait prior to timing out.
716
- """
717
- ...
718
-
719
- @typing.overload
720
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
721
- """
722
- Creates a human-readable report, a Metaflow Card, after this step completes.
723
-
724
- Note that you may add multiple `@card` decorators in a step with different parameters.
725
-
726
-
727
- Parameters
728
- ----------
729
- type : str, default 'default'
730
- Card type.
731
- id : str, optional, default None
732
- If multiple cards are present, use this id to identify this card.
733
- options : Dict[str, Any], default {}
734
- Options passed to the card. The contents depend on the card type.
735
- timeout : int, default 45
736
- Interrupt reporting if it takes more than this many seconds.
737
- """
738
- ...
739
-
740
- @typing.overload
741
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
742
- ...
743
-
744
- @typing.overload
745
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
746
- ...
747
-
748
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
749
- """
750
- Creates a human-readable report, a Metaflow Card, after this step completes.
751
-
752
- Note that you may add multiple `@card` decorators in a step with different parameters.
753
-
754
-
755
- Parameters
756
- ----------
757
- type : str, default 'default'
758
- Card type.
759
- id : str, optional, default None
760
- If multiple cards are present, use this id to identify this card.
761
- options : Dict[str, Any], default {}
762
- Options passed to the card. The contents depend on the card type.
763
- timeout : int, default 45
764
- Interrupt reporting if it takes more than this many seconds.
765
- """
766
- ...
767
-
768
- @typing.overload
769
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
770
- """
771
- Enables checkpointing for a step.
616
+ Enables checkpointing for a step.
772
617
 
773
618
 
774
619
 
@@ -822,393 +667,529 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
822
667
  """
823
668
  ...
824
669
 
670
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
671
+ """
672
+ Specifies that this step is used to deploy an instance of the app.
673
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
674
+
675
+
676
+ Parameters
677
+ ----------
678
+ app_port : int
679
+ Number of GPUs to use.
680
+ app_name : str
681
+ Name of the app to deploy.
682
+ """
683
+ ...
684
+
825
685
  @typing.overload
826
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
686
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
827
687
  """
828
- Specifies the Conda environment for the step.
688
+ Enables loading / saving of models within a step.
829
689
 
830
- Information in this decorator will augment any
831
- attributes set in the `@conda_base` flow-level decorator. Hence,
832
- you can use `@conda_base` to set packages required by all
833
- steps and use `@conda` to specify step-specific overrides.
834
690
 
835
691
 
836
692
  Parameters
837
693
  ----------
838
- packages : Dict[str, str], default {}
839
- Packages to use for this step. The key is the name of the package
840
- and the value is the version to use.
841
- libraries : Dict[str, str], default {}
842
- Supported for backward compatibility. When used with packages, packages will take precedence.
843
- python : str, optional, default None
844
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
845
- that the version used will correspond to the version of the Python interpreter used to start the run.
846
- disabled : bool, default False
847
- If set to True, disables @conda.
694
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
695
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
696
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
697
+ - `current.checkpoint`
698
+ - `current.model`
699
+ - `current.huggingface_hub`
700
+
701
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
702
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
703
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
704
+
705
+ temp_dir_root : str, default: None
706
+ The root directory under which `current.model.loaded` will store loaded models
848
707
  """
849
708
  ...
850
709
 
851
710
  @typing.overload
852
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
711
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
853
712
  ...
854
713
 
855
714
  @typing.overload
856
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
715
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
857
716
  ...
858
717
 
859
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
718
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
860
719
  """
861
- Specifies the Conda environment for the step.
720
+ Enables loading / saving of models within a step.
862
721
 
863
- Information in this decorator will augment any
864
- attributes set in the `@conda_base` flow-level decorator. Hence,
865
- you can use `@conda_base` to set packages required by all
866
- steps and use `@conda` to specify step-specific overrides.
867
722
 
868
723
 
869
724
  Parameters
870
725
  ----------
871
- packages : Dict[str, str], default {}
872
- Packages to use for this step. The key is the name of the package
873
- and the value is the version to use.
874
- libraries : Dict[str, str], default {}
875
- Supported for backward compatibility. When used with packages, packages will take precedence.
876
- python : str, optional, default None
877
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
878
- that the version used will correspond to the version of the Python interpreter used to start the run.
879
- disabled : bool, default False
880
- If set to True, disables @conda.
726
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
727
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
728
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
729
+ - `current.checkpoint`
730
+ - `current.model`
731
+ - `current.huggingface_hub`
732
+
733
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
734
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
735
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
736
+
737
+ temp_dir_root : str, default: None
738
+ The root directory under which `current.model.loaded` will store loaded models
881
739
  """
882
740
  ...
883
741
 
884
742
  @typing.overload
885
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
743
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
886
744
  """
887
- Specifies secrets to be retrieved and injected as environment variables prior to
888
- the execution of a step.
745
+ Specifies that the step will success under all circumstances.
746
+
747
+ The decorator will create an optional artifact, specified by `var`, which
748
+ contains the exception raised. You can use it to detect the presence
749
+ of errors, indicating that all happy-path artifacts produced by the step
750
+ are missing.
889
751
 
890
752
 
891
753
  Parameters
892
754
  ----------
893
- sources : List[Union[str, Dict[str, Any]]], default: []
894
- List of secret specs, defining how the secrets are to be retrieved
755
+ var : str, optional, default None
756
+ Name of the artifact in which to store the caught exception.
757
+ If not specified, the exception is not stored.
758
+ print_exception : bool, default True
759
+ Determines whether or not the exception is printed to
760
+ stdout when caught.
895
761
  """
896
762
  ...
897
763
 
898
764
  @typing.overload
899
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
765
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
900
766
  ...
901
767
 
902
768
  @typing.overload
903
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
769
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
904
770
  ...
905
771
 
906
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
772
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
907
773
  """
908
- Specifies secrets to be retrieved and injected as environment variables prior to
909
- the execution of a step.
774
+ Specifies that the step will success under all circumstances.
775
+
776
+ The decorator will create an optional artifact, specified by `var`, which
777
+ contains the exception raised. You can use it to detect the presence
778
+ of errors, indicating that all happy-path artifacts produced by the step
779
+ are missing.
910
780
 
911
781
 
912
782
  Parameters
913
783
  ----------
914
- sources : List[Union[str, Dict[str, Any]]], default: []
915
- List of secret specs, defining how the secrets are to be retrieved
784
+ var : str, optional, default None
785
+ Name of the artifact in which to store the caught exception.
786
+ If not specified, the exception is not stored.
787
+ print_exception : bool, default True
788
+ Determines whether or not the exception is printed to
789
+ stdout when caught.
916
790
  """
917
791
  ...
918
792
 
919
793
  @typing.overload
920
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
794
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
921
795
  """
922
- Specifies the PyPI packages for the step.
796
+ Specifies the number of times the task corresponding
797
+ to a step needs to be retried.
923
798
 
924
- Information in this decorator will augment any
925
- attributes set in the `@pyi_base` flow-level decorator. Hence,
926
- you can use `@pypi_base` to set packages required by all
927
- steps and use `@pypi` to specify step-specific overrides.
799
+ This decorator is useful for handling transient errors, such as networking issues.
800
+ If your task contains operations that can't be retried safely, e.g. database updates,
801
+ it is advisable to annotate it with `@retry(times=0)`.
802
+
803
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
804
+ decorator will execute a no-op task after all retries have been exhausted,
805
+ ensuring that the flow execution can continue.
928
806
 
929
807
 
930
808
  Parameters
931
809
  ----------
932
- packages : Dict[str, str], default: {}
933
- Packages to use for this step. The key is the name of the package
934
- and the value is the version to use.
935
- python : str, optional, default: None
936
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
937
- that the version used will correspond to the version of the Python interpreter used to start the run.
810
+ times : int, default 3
811
+ Number of times to retry this task.
812
+ minutes_between_retries : int, default 2
813
+ Number of minutes between retries.
938
814
  """
939
815
  ...
940
816
 
941
817
  @typing.overload
942
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
818
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
943
819
  ...
944
820
 
945
821
  @typing.overload
946
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
822
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
947
823
  ...
948
824
 
949
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
825
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
950
826
  """
951
- Specifies the PyPI packages for the step.
827
+ Specifies the number of times the task corresponding
828
+ to a step needs to be retried.
952
829
 
953
- Information in this decorator will augment any
954
- attributes set in the `@pyi_base` flow-level decorator. Hence,
955
- you can use `@pypi_base` to set packages required by all
956
- steps and use `@pypi` to specify step-specific overrides.
830
+ This decorator is useful for handling transient errors, such as networking issues.
831
+ If your task contains operations that can't be retried safely, e.g. database updates,
832
+ it is advisable to annotate it with `@retry(times=0)`.
833
+
834
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
835
+ decorator will execute a no-op task after all retries have been exhausted,
836
+ ensuring that the flow execution can continue.
957
837
 
958
838
 
959
839
  Parameters
960
840
  ----------
961
- packages : Dict[str, str], default: {}
962
- Packages to use for this step. The key is the name of the package
963
- and the value is the version to use.
964
- python : str, optional, default: None
965
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
966
- that the version used will correspond to the version of the Python interpreter used to start the run.
841
+ times : int, default 3
842
+ Number of times to retry this task.
843
+ minutes_between_retries : int, default 2
844
+ Number of minutes between retries.
967
845
  """
968
846
  ...
969
847
 
970
- @typing.overload
971
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
848
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
972
849
  """
973
- Specifies the event(s) that this flow depends on.
850
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
974
851
 
975
- ```
976
- @trigger(event='foo')
977
- ```
978
- or
979
- ```
980
- @trigger(events=['foo', 'bar'])
981
- ```
852
+ User code call
853
+ --------------
854
+ @ollama(
855
+ models=[...],
856
+ ...
857
+ )
982
858
 
983
- Additionally, you can specify the parameter mappings
984
- to map event payload to Metaflow parameters for the flow.
985
- ```
986
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
987
- ```
988
- or
989
- ```
990
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
991
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
992
- ```
859
+ Valid backend options
860
+ ---------------------
861
+ - 'local': Run as a separate process on the local task machine.
862
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
863
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
993
864
 
994
- 'parameters' can also be a list of strings and tuples like so:
995
- ```
996
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
997
- ```
998
- This is equivalent to:
999
- ```
1000
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1001
- ```
865
+ Valid model options
866
+ -------------------
867
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
1002
868
 
1003
869
 
1004
870
  Parameters
1005
871
  ----------
1006
- event : Union[str, Dict[str, Any]], optional, default None
1007
- Event dependency for this flow.
1008
- events : List[Union[str, Dict[str, Any]]], default []
1009
- Events dependency for this flow.
1010
- options : Dict[str, Any], default {}
1011
- Backend-specific configuration for tuning eventing behavior.
872
+ models: list[str]
873
+ List of Ollama containers running models in sidecars.
874
+ backend: str
875
+ Determines where and how to run the Ollama process.
876
+ force_pull: bool
877
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
878
+ cache_update_policy: str
879
+ Cache update policy: "auto", "force", or "never".
880
+ force_cache_update: bool
881
+ Simple override for "force" cache update policy.
882
+ debug: bool
883
+ Whether to turn on verbose debugging logs.
884
+ circuit_breaker_config: dict
885
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
886
+ timeout_config: dict
887
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
1012
888
  """
1013
889
  ...
1014
890
 
1015
- @typing.overload
1016
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
891
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
892
+ """
893
+ Specifies that this step should execute on Kubernetes.
894
+
895
+
896
+ Parameters
897
+ ----------
898
+ cpu : int, default 1
899
+ Number of CPUs required for this step. If `@resources` is
900
+ also present, the maximum value from all decorators is used.
901
+ memory : int, default 4096
902
+ Memory size (in MB) required for this step. If
903
+ `@resources` is also present, the maximum value from all decorators is
904
+ used.
905
+ disk : int, default 10240
906
+ Disk size (in MB) required for this step. If
907
+ `@resources` is also present, the maximum value from all decorators is
908
+ used.
909
+ image : str, optional, default None
910
+ Docker image to use when launching on Kubernetes. If not specified, and
911
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
912
+ not, a default Docker image mapping to the current version of Python is used.
913
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
914
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
915
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
916
+ Kubernetes service account to use when launching pod in Kubernetes.
917
+ secrets : List[str], optional, default None
918
+ Kubernetes secrets to use when launching pod in Kubernetes. These
919
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
920
+ in Metaflow configuration.
921
+ node_selector: Union[Dict[str,str], str], optional, default None
922
+ Kubernetes node selector(s) to apply to the pod running the task.
923
+ Can be passed in as a comma separated string of values e.g.
924
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
925
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
926
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
927
+ Kubernetes namespace to use when launching pod in Kubernetes.
928
+ gpu : int, optional, default None
929
+ Number of GPUs required for this step. A value of zero implies that
930
+ the scheduled node should not have GPUs.
931
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
932
+ The vendor of the GPUs to be used for this step.
933
+ tolerations : List[str], default []
934
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
935
+ Kubernetes tolerations to use when launching pod in Kubernetes.
936
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
937
+ Kubernetes labels to use when launching pod in Kubernetes.
938
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
939
+ Kubernetes annotations to use when launching pod in Kubernetes.
940
+ use_tmpfs : bool, default False
941
+ This enables an explicit tmpfs mount for this step.
942
+ tmpfs_tempdir : bool, default True
943
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
944
+ tmpfs_size : int, optional, default: None
945
+ The value for the size (in MiB) of the tmpfs mount for this step.
946
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
947
+ memory allocated for this step.
948
+ tmpfs_path : str, optional, default /metaflow_temp
949
+ Path to tmpfs mount for this step.
950
+ persistent_volume_claims : Dict[str, str], optional, default None
951
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
952
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
953
+ shared_memory: int, optional
954
+ Shared memory size (in MiB) required for this step
955
+ port: int, optional
956
+ Port number to specify in the Kubernetes job object
957
+ compute_pool : str, optional, default None
958
+ Compute pool to be used for for this step.
959
+ If not specified, any accessible compute pool within the perimeter is used.
960
+ hostname_resolution_timeout: int, default 10 * 60
961
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
962
+ Only applicable when @parallel is used.
963
+ qos: str, default: Burstable
964
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
965
+
966
+ security_context: Dict[str, Any], optional, default None
967
+ Container security context. Applies to the task container. Allows the following keys:
968
+ - privileged: bool, optional, default None
969
+ - allow_privilege_escalation: bool, optional, default None
970
+ - run_as_user: int, optional, default None
971
+ - run_as_group: int, optional, default None
972
+ - run_as_non_root: bool, optional, default None
973
+ """
1017
974
  ...
1018
975
 
1019
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
976
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1020
977
  """
1021
- Specifies the event(s) that this flow depends on.
978
+ Specifies what flows belong to the same project.
1022
979
 
1023
- ```
1024
- @trigger(event='foo')
1025
- ```
1026
- or
1027
- ```
1028
- @trigger(events=['foo', 'bar'])
1029
- ```
980
+ A project-specific namespace is created for all flows that
981
+ use the same `@project(name)`.
1030
982
 
1031
- Additionally, you can specify the parameter mappings
1032
- to map event payload to Metaflow parameters for the flow.
1033
- ```
1034
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1035
- ```
1036
- or
1037
- ```
1038
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1039
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1040
- ```
1041
983
 
1042
- 'parameters' can also be a list of strings and tuples like so:
1043
- ```
1044
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1045
- ```
1046
- This is equivalent to:
1047
- ```
1048
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1049
- ```
984
+ Parameters
985
+ ----------
986
+ name : str
987
+ Project name. Make sure that the name is unique amongst all
988
+ projects that use the same production scheduler. The name may
989
+ contain only lowercase alphanumeric characters and underscores.
990
+
991
+ branch : Optional[str], default None
992
+ The branch to use. If not specified, the branch is set to
993
+ `user.<username>` unless `production` is set to `True`. This can
994
+ also be set on the command line using `--branch` as a top-level option.
995
+ It is an error to specify `branch` in the decorator and on the command line.
996
+
997
+ production : bool, default False
998
+ Whether or not the branch is the production branch. This can also be set on the
999
+ command line using `--production` as a top-level option. It is an error to specify
1000
+ `production` in the decorator and on the command line.
1001
+ The project branch name will be:
1002
+ - if `branch` is specified:
1003
+ - if `production` is True: `prod.<branch>`
1004
+ - if `production` is False: `test.<branch>`
1005
+ - if `branch` is not specified:
1006
+ - if `production` is True: `prod`
1007
+ - if `production` is False: `user.<username>`
1008
+ """
1009
+ ...
1010
+
1011
+ @typing.overload
1012
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1013
+ """
1014
+ Specifies the Conda environment for all steps of the flow.
1015
+
1016
+ Use `@conda_base` to set common libraries required by all
1017
+ steps and use `@conda` to specify step-specific additions.
1050
1018
 
1051
1019
 
1052
1020
  Parameters
1053
1021
  ----------
1054
- event : Union[str, Dict[str, Any]], optional, default None
1055
- Event dependency for this flow.
1056
- events : List[Union[str, Dict[str, Any]]], default []
1057
- Events dependency for this flow.
1058
- options : Dict[str, Any], default {}
1059
- Backend-specific configuration for tuning eventing behavior.
1022
+ packages : Dict[str, str], default {}
1023
+ Packages to use for this flow. The key is the name of the package
1024
+ and the value is the version to use.
1025
+ libraries : Dict[str, str], default {}
1026
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1027
+ python : str, optional, default None
1028
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1029
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1030
+ disabled : bool, default False
1031
+ If set to True, disables Conda.
1060
1032
  """
1061
1033
  ...
1062
1034
 
1063
1035
  @typing.overload
1064
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1036
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1037
+ ...
1038
+
1039
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1065
1040
  """
1066
- Specifies the flow(s) that this flow depends on.
1067
-
1068
- ```
1069
- @trigger_on_finish(flow='FooFlow')
1070
- ```
1071
- or
1072
- ```
1073
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1074
- ```
1075
- This decorator respects the @project decorator and triggers the flow
1076
- when upstream runs within the same namespace complete successfully
1041
+ Specifies the Conda environment for all steps of the flow.
1077
1042
 
1078
- Additionally, you can specify project aware upstream flow dependencies
1079
- by specifying the fully qualified project_flow_name.
1080
- ```
1081
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1082
- ```
1083
- or
1084
- ```
1085
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1086
- ```
1043
+ Use `@conda_base` to set common libraries required by all
1044
+ steps and use `@conda` to specify step-specific additions.
1087
1045
 
1088
- You can also specify just the project or project branch (other values will be
1089
- inferred from the current project or project branch):
1090
- ```
1091
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1092
- ```
1093
1046
 
1094
- Note that `branch` is typically one of:
1095
- - `prod`
1096
- - `user.bob`
1097
- - `test.my_experiment`
1098
- - `prod.staging`
1047
+ Parameters
1048
+ ----------
1049
+ packages : Dict[str, str], default {}
1050
+ Packages to use for this flow. The key is the name of the package
1051
+ and the value is the version to use.
1052
+ libraries : Dict[str, str], default {}
1053
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1054
+ python : str, optional, default None
1055
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1056
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1057
+ disabled : bool, default False
1058
+ If set to True, disables Conda.
1059
+ """
1060
+ ...
1061
+
1062
+ @typing.overload
1063
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1064
+ """
1065
+ Specifies the PyPI packages for all steps of the flow.
1099
1066
 
1067
+ Use `@pypi_base` to set common packages required by all
1068
+ steps and use `@pypi` to specify step-specific overrides.
1100
1069
 
1101
1070
  Parameters
1102
1071
  ----------
1103
- flow : Union[str, Dict[str, str]], optional, default None
1104
- Upstream flow dependency for this flow.
1105
- flows : List[Union[str, Dict[str, str]]], default []
1106
- Upstream flow dependencies for this flow.
1107
- options : Dict[str, Any], default {}
1108
- Backend-specific configuration for tuning eventing behavior.
1072
+ packages : Dict[str, str], default: {}
1073
+ Packages to use for this flow. The key is the name of the package
1074
+ and the value is the version to use.
1075
+ python : str, optional, default: None
1076
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1077
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1109
1078
  """
1110
1079
  ...
1111
1080
 
1112
1081
  @typing.overload
1113
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1082
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1114
1083
  ...
1115
1084
 
1116
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1085
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1117
1086
  """
1118
- Specifies the flow(s) that this flow depends on.
1087
+ Specifies the PyPI packages for all steps of the flow.
1088
+
1089
+ Use `@pypi_base` to set common packages required by all
1090
+ steps and use `@pypi` to specify step-specific overrides.
1091
+
1092
+ Parameters
1093
+ ----------
1094
+ packages : Dict[str, str], default: {}
1095
+ Packages to use for this flow. The key is the name of the package
1096
+ and the value is the version to use.
1097
+ python : str, optional, default: None
1098
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1099
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1100
+ """
1101
+ ...
1102
+
1103
+ @typing.overload
1104
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1105
+ """
1106
+ Specifies the event(s) that this flow depends on.
1119
1107
 
1120
1108
  ```
1121
- @trigger_on_finish(flow='FooFlow')
1109
+ @trigger(event='foo')
1122
1110
  ```
1123
1111
  or
1124
1112
  ```
1125
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1113
+ @trigger(events=['foo', 'bar'])
1126
1114
  ```
1127
- This decorator respects the @project decorator and triggers the flow
1128
- when upstream runs within the same namespace complete successfully
1129
1115
 
1130
- Additionally, you can specify project aware upstream flow dependencies
1131
- by specifying the fully qualified project_flow_name.
1116
+ Additionally, you can specify the parameter mappings
1117
+ to map event payload to Metaflow parameters for the flow.
1132
1118
  ```
1133
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1119
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1134
1120
  ```
1135
1121
  or
1136
1122
  ```
1137
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1138
- ```
1139
-
1140
- You can also specify just the project or project branch (other values will be
1141
- inferred from the current project or project branch):
1142
- ```
1143
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1144
- ```
1145
-
1146
- Note that `branch` is typically one of:
1147
- - `prod`
1148
- - `user.bob`
1149
- - `test.my_experiment`
1150
- - `prod.staging`
1151
-
1152
-
1153
- Parameters
1154
- ----------
1155
- flow : Union[str, Dict[str, str]], optional, default None
1156
- Upstream flow dependency for this flow.
1157
- flows : List[Union[str, Dict[str, str]]], default []
1158
- Upstream flow dependencies for this flow.
1159
- options : Dict[str, Any], default {}
1160
- Backend-specific configuration for tuning eventing behavior.
1161
- """
1162
- ...
1163
-
1164
- @typing.overload
1165
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1166
- """
1167
- Specifies the Conda environment for all steps of the flow.
1123
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1124
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1125
+ ```
1168
1126
 
1169
- Use `@conda_base` to set common libraries required by all
1170
- steps and use `@conda` to specify step-specific additions.
1127
+ 'parameters' can also be a list of strings and tuples like so:
1128
+ ```
1129
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1130
+ ```
1131
+ This is equivalent to:
1132
+ ```
1133
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1134
+ ```
1171
1135
 
1172
1136
 
1173
1137
  Parameters
1174
1138
  ----------
1175
- packages : Dict[str, str], default {}
1176
- Packages to use for this flow. The key is the name of the package
1177
- and the value is the version to use.
1178
- libraries : Dict[str, str], default {}
1179
- Supported for backward compatibility. When used with packages, packages will take precedence.
1180
- python : str, optional, default None
1181
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1182
- that the version used will correspond to the version of the Python interpreter used to start the run.
1183
- disabled : bool, default False
1184
- If set to True, disables Conda.
1139
+ event : Union[str, Dict[str, Any]], optional, default None
1140
+ Event dependency for this flow.
1141
+ events : List[Union[str, Dict[str, Any]]], default []
1142
+ Events dependency for this flow.
1143
+ options : Dict[str, Any], default {}
1144
+ Backend-specific configuration for tuning eventing behavior.
1185
1145
  """
1186
1146
  ...
1187
1147
 
1188
1148
  @typing.overload
1189
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1149
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1190
1150
  ...
1191
1151
 
1192
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1152
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1193
1153
  """
1194
- Specifies the Conda environment for all steps of the flow.
1154
+ Specifies the event(s) that this flow depends on.
1195
1155
 
1196
- Use `@conda_base` to set common libraries required by all
1197
- steps and use `@conda` to specify step-specific additions.
1156
+ ```
1157
+ @trigger(event='foo')
1158
+ ```
1159
+ or
1160
+ ```
1161
+ @trigger(events=['foo', 'bar'])
1162
+ ```
1163
+
1164
+ Additionally, you can specify the parameter mappings
1165
+ to map event payload to Metaflow parameters for the flow.
1166
+ ```
1167
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1168
+ ```
1169
+ or
1170
+ ```
1171
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1172
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1173
+ ```
1174
+
1175
+ 'parameters' can also be a list of strings and tuples like so:
1176
+ ```
1177
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1178
+ ```
1179
+ This is equivalent to:
1180
+ ```
1181
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1182
+ ```
1198
1183
 
1199
1184
 
1200
1185
  Parameters
1201
1186
  ----------
1202
- packages : Dict[str, str], default {}
1203
- Packages to use for this flow. The key is the name of the package
1204
- and the value is the version to use.
1205
- libraries : Dict[str, str], default {}
1206
- Supported for backward compatibility. When used with packages, packages will take precedence.
1207
- python : str, optional, default None
1208
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1209
- that the version used will correspond to the version of the Python interpreter used to start the run.
1210
- disabled : bool, default False
1211
- If set to True, disables Conda.
1187
+ event : Union[str, Dict[str, Any]], optional, default None
1188
+ Event dependency for this flow.
1189
+ events : List[Union[str, Dict[str, Any]]], default []
1190
+ Events dependency for this flow.
1191
+ options : Dict[str, Any], default {}
1192
+ Backend-specific configuration for tuning eventing behavior.
1212
1193
  """
1213
1194
  ...
1214
1195
 
@@ -1306,82 +1287,6 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1306
1287
  """
1307
1288
  ...
1308
1289
 
1309
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1310
- """
1311
- Specifies what flows belong to the same project.
1312
-
1313
- A project-specific namespace is created for all flows that
1314
- use the same `@project(name)`.
1315
-
1316
-
1317
- Parameters
1318
- ----------
1319
- name : str
1320
- Project name. Make sure that the name is unique amongst all
1321
- projects that use the same production scheduler. The name may
1322
- contain only lowercase alphanumeric characters and underscores.
1323
-
1324
- branch : Optional[str], default None
1325
- The branch to use. If not specified, the branch is set to
1326
- `user.<username>` unless `production` is set to `True`. This can
1327
- also be set on the command line using `--branch` as a top-level option.
1328
- It is an error to specify `branch` in the decorator and on the command line.
1329
-
1330
- production : bool, default False
1331
- Whether or not the branch is the production branch. This can also be set on the
1332
- command line using `--production` as a top-level option. It is an error to specify
1333
- `production` in the decorator and on the command line.
1334
- The project branch name will be:
1335
- - if `branch` is specified:
1336
- - if `production` is True: `prod.<branch>`
1337
- - if `production` is False: `test.<branch>`
1338
- - if `branch` is not specified:
1339
- - if `production` is True: `prod`
1340
- - if `production` is False: `user.<username>`
1341
- """
1342
- ...
1343
-
1344
- @typing.overload
1345
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1346
- """
1347
- Specifies the PyPI packages for all steps of the flow.
1348
-
1349
- Use `@pypi_base` to set common packages required by all
1350
- steps and use `@pypi` to specify step-specific overrides.
1351
-
1352
- Parameters
1353
- ----------
1354
- packages : Dict[str, str], default: {}
1355
- Packages to use for this flow. The key is the name of the package
1356
- and the value is the version to use.
1357
- python : str, optional, default: None
1358
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1359
- that the version used will correspond to the version of the Python interpreter used to start the run.
1360
- """
1361
- ...
1362
-
1363
- @typing.overload
1364
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1365
- ...
1366
-
1367
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1368
- """
1369
- Specifies the PyPI packages for all steps of the flow.
1370
-
1371
- Use `@pypi_base` to set common packages required by all
1372
- steps and use `@pypi` to specify step-specific overrides.
1373
-
1374
- Parameters
1375
- ----------
1376
- packages : Dict[str, str], default: {}
1377
- Packages to use for this flow. The key is the name of the package
1378
- and the value is the version to use.
1379
- python : str, optional, default: None
1380
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1381
- that the version used will correspond to the version of the Python interpreter used to start the run.
1382
- """
1383
- ...
1384
-
1385
1290
  def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1386
1291
  """
1387
1292
  Allows setting external datastores to save data for the
@@ -1539,5 +1444,106 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1539
1444
  """
1540
1445
  ...
1541
1446
 
1447
+ @typing.overload
1448
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1449
+ """
1450
+ Specifies the flow(s) that this flow depends on.
1451
+
1452
+ ```
1453
+ @trigger_on_finish(flow='FooFlow')
1454
+ ```
1455
+ or
1456
+ ```
1457
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1458
+ ```
1459
+ This decorator respects the @project decorator and triggers the flow
1460
+ when upstream runs within the same namespace complete successfully
1461
+
1462
+ Additionally, you can specify project aware upstream flow dependencies
1463
+ by specifying the fully qualified project_flow_name.
1464
+ ```
1465
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1466
+ ```
1467
+ or
1468
+ ```
1469
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1470
+ ```
1471
+
1472
+ You can also specify just the project or project branch (other values will be
1473
+ inferred from the current project or project branch):
1474
+ ```
1475
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1476
+ ```
1477
+
1478
+ Note that `branch` is typically one of:
1479
+ - `prod`
1480
+ - `user.bob`
1481
+ - `test.my_experiment`
1482
+ - `prod.staging`
1483
+
1484
+
1485
+ Parameters
1486
+ ----------
1487
+ flow : Union[str, Dict[str, str]], optional, default None
1488
+ Upstream flow dependency for this flow.
1489
+ flows : List[Union[str, Dict[str, str]]], default []
1490
+ Upstream flow dependencies for this flow.
1491
+ options : Dict[str, Any], default {}
1492
+ Backend-specific configuration for tuning eventing behavior.
1493
+ """
1494
+ ...
1495
+
1496
+ @typing.overload
1497
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1498
+ ...
1499
+
1500
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1501
+ """
1502
+ Specifies the flow(s) that this flow depends on.
1503
+
1504
+ ```
1505
+ @trigger_on_finish(flow='FooFlow')
1506
+ ```
1507
+ or
1508
+ ```
1509
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1510
+ ```
1511
+ This decorator respects the @project decorator and triggers the flow
1512
+ when upstream runs within the same namespace complete successfully
1513
+
1514
+ Additionally, you can specify project aware upstream flow dependencies
1515
+ by specifying the fully qualified project_flow_name.
1516
+ ```
1517
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1518
+ ```
1519
+ or
1520
+ ```
1521
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1522
+ ```
1523
+
1524
+ You can also specify just the project or project branch (other values will be
1525
+ inferred from the current project or project branch):
1526
+ ```
1527
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1528
+ ```
1529
+
1530
+ Note that `branch` is typically one of:
1531
+ - `prod`
1532
+ - `user.bob`
1533
+ - `test.my_experiment`
1534
+ - `prod.staging`
1535
+
1536
+
1537
+ Parameters
1538
+ ----------
1539
+ flow : Union[str, Dict[str, str]], optional, default None
1540
+ Upstream flow dependency for this flow.
1541
+ flows : List[Union[str, Dict[str, str]]], default []
1542
+ Upstream flow dependencies for this flow.
1543
+ options : Dict[str, Any], default {}
1544
+ Backend-specific configuration for tuning eventing behavior.
1545
+ """
1546
+ ...
1547
+
1542
1548
  pkg_name: str
1543
1549