ob-metaflow-stubs 6.0.3.177__py2.py3-none-any.whl → 6.0.3.179__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. metaflow-stubs/__init__.pyi +836 -794
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +27 -27
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +5 -5
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +6 -6
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +4 -2
  20. metaflow-stubs/metaflow_current.pyi +102 -102
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +4 -4
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +5 -5
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +5 -5
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +2 -2
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  83. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  84. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  85. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  86. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  87. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  88. metaflow-stubs/multicore_utils.pyi +2 -2
  89. metaflow-stubs/parameters.pyi +5 -5
  90. metaflow-stubs/plugins/__init__.pyi +11 -11
  91. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  92. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  93. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  94. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  96. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  97. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  98. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  100. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  101. metaflow-stubs/plugins/argo/argo_workflows.pyi +6 -4
  102. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  103. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  104. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -6
  105. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  106. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  107. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  108. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  110. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  111. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  112. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  113. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  114. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  116. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  118. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  119. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  120. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +4 -4
  121. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  122. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  123. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  124. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  125. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  126. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  127. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  128. metaflow-stubs/plugins/cards/__init__.pyi +6 -6
  129. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  130. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  131. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  132. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  133. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  134. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  135. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  136. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  137. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  138. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  139. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  140. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  141. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  142. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  143. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  144. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  146. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  147. metaflow-stubs/plugins/datatools/s3/s3.pyi +15 -15
  148. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  149. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  150. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  151. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  152. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  153. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  154. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  155. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  156. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  157. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  158. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  159. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  160. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  161. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  162. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  163. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  164. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +4 -4
  165. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  166. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +8 -2
  167. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  168. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  169. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  170. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  171. metaflow-stubs/plugins/perimeters.pyi +2 -2
  172. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  173. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  174. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  175. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  176. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  177. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  178. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  179. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  180. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  181. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  182. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  183. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  184. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  185. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  186. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  187. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  188. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  189. metaflow-stubs/plugins/torchtune/__init__.pyi +2 -2
  190. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  191. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  192. metaflow-stubs/profilers/__init__.pyi +2 -2
  193. metaflow-stubs/pylint_wrapper.pyi +2 -2
  194. metaflow-stubs/runner/__init__.pyi +2 -2
  195. metaflow-stubs/runner/deployer.pyi +33 -30
  196. metaflow-stubs/runner/deployer_impl.pyi +6 -3
  197. metaflow-stubs/runner/metaflow_runner.pyi +8 -5
  198. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  199. metaflow-stubs/runner/nbrun.pyi +2 -2
  200. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  201. metaflow-stubs/runner/utils.pyi +5 -2
  202. metaflow-stubs/system/__init__.pyi +2 -2
  203. metaflow-stubs/system/system_logger.pyi +2 -2
  204. metaflow-stubs/system/system_monitor.pyi +2 -2
  205. metaflow-stubs/tagging_util.pyi +2 -2
  206. metaflow-stubs/tuple_util.pyi +2 -2
  207. metaflow-stubs/user_configs/__init__.pyi +2 -2
  208. metaflow-stubs/user_configs/config_decorators.pyi +8 -8
  209. metaflow-stubs/user_configs/config_options.pyi +3 -3
  210. metaflow-stubs/user_configs/config_parameters.pyi +77 -17
  211. {ob_metaflow_stubs-6.0.3.177.dist-info → ob_metaflow_stubs-6.0.3.179.dist-info}/METADATA +1 -1
  212. ob_metaflow_stubs-6.0.3.179.dist-info/RECORD +215 -0
  213. ob_metaflow_stubs-6.0.3.177.dist-info/RECORD +0 -215
  214. {ob_metaflow_stubs-6.0.3.177.dist-info → ob_metaflow_stubs-6.0.3.179.dist-info}/WHEEL +0 -0
  215. {ob_metaflow_stubs-6.0.3.177.dist-info → ob_metaflow_stubs-6.0.3.179.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.14.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-06-10T19:56:51.889454 #
3
+ # MF version: 2.15.17.1+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-06-16T20:11:03.038176 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -35,8 +35,8 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
- from . import cards as cards
39
38
  from . import tuple_util as tuple_util
39
+ from . import cards as cards
40
40
  from . import metaflow_git as metaflow_git
41
41
  from . import events as events
42
42
  from . import runner as runner
@@ -44,9 +44,9 @@ from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
- from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
48
47
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
49
48
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
49
+ from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
52
52
  from .client.core import get_namespace as get_namespace
@@ -155,228 +155,59 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
155
155
  ...
156
156
 
157
157
  @typing.overload
158
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
159
- """
160
- Creates a human-readable report, a Metaflow Card, after this step completes.
161
-
162
- Note that you may add multiple `@card` decorators in a step with different parameters.
163
-
164
-
165
- Parameters
166
- ----------
167
- type : str, default 'default'
168
- Card type.
169
- id : str, optional, default None
170
- If multiple cards are present, use this id to identify this card.
171
- options : Dict[str, Any], default {}
172
- Options passed to the card. The contents depend on the card type.
173
- timeout : int, default 45
174
- Interrupt reporting if it takes more than this many seconds.
175
- """
176
- ...
177
-
178
- @typing.overload
179
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
180
- ...
181
-
182
- @typing.overload
183
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
184
- ...
185
-
186
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
158
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
187
159
  """
188
- Creates a human-readable report, a Metaflow Card, after this step completes.
160
+ Enables loading / saving of models within a step.
189
161
 
190
- Note that you may add multiple `@card` decorators in a step with different parameters.
191
162
 
192
163
 
193
164
  Parameters
194
165
  ----------
195
- type : str, default 'default'
196
- Card type.
197
- id : str, optional, default None
198
- If multiple cards are present, use this id to identify this card.
199
- options : Dict[str, Any], default {}
200
- Options passed to the card. The contents depend on the card type.
201
- timeout : int, default 45
202
- Interrupt reporting if it takes more than this many seconds.
203
- """
204
- ...
205
-
206
- @typing.overload
207
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
208
- """
209
- Decorator prototype for all step decorators. This function gets specialized
210
- and imported for all decorators types by _import_plugin_decorators().
211
- """
212
- ...
213
-
214
- @typing.overload
215
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
216
- ...
217
-
218
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
219
- """
220
- Decorator prototype for all step decorators. This function gets specialized
221
- and imported for all decorators types by _import_plugin_decorators().
222
- """
223
- ...
224
-
225
- @typing.overload
226
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
227
- """
228
- Specifies a timeout for your step.
229
-
230
- This decorator is useful if this step may hang indefinitely.
231
-
232
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
233
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
234
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
235
-
236
- Note that all the values specified in parameters are added together so if you specify
237
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
166
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
167
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
168
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
169
+ - `current.checkpoint`
170
+ - `current.model`
171
+ - `current.huggingface_hub`
238
172
 
173
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
174
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
175
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
239
176
 
240
- Parameters
241
- ----------
242
- seconds : int, default 0
243
- Number of seconds to wait prior to timing out.
244
- minutes : int, default 0
245
- Number of minutes to wait prior to timing out.
246
- hours : int, default 0
247
- Number of hours to wait prior to timing out.
177
+ temp_dir_root : str, default: None
178
+ The root directory under which `current.model.loaded` will store loaded models
248
179
  """
249
180
  ...
250
181
 
251
182
  @typing.overload
252
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
183
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
253
184
  ...
254
185
 
255
186
  @typing.overload
256
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
187
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
257
188
  ...
258
189
 
259
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
190
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
260
191
  """
261
- Specifies a timeout for your step.
262
-
263
- This decorator is useful if this step may hang indefinitely.
264
-
265
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
266
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
267
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
268
-
269
- Note that all the values specified in parameters are added together so if you specify
270
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
271
-
192
+ Enables loading / saving of models within a step.
272
193
 
273
- Parameters
274
- ----------
275
- seconds : int, default 0
276
- Number of seconds to wait prior to timing out.
277
- minutes : int, default 0
278
- Number of minutes to wait prior to timing out.
279
- hours : int, default 0
280
- Number of hours to wait prior to timing out.
281
- """
282
- ...
283
-
284
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
285
- """
286
- Specifies that this step should execute on DGX cloud.
287
194
 
288
195
 
289
196
  Parameters
290
197
  ----------
291
- gpu : int
292
- Number of GPUs to use.
293
- gpu_type : str
294
- Type of Nvidia GPU to use.
295
- """
296
- ...
297
-
298
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
299
- """
300
- Specifies that this step should execute on Kubernetes.
301
-
198
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
199
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
200
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
201
+ - `current.checkpoint`
202
+ - `current.model`
203
+ - `current.huggingface_hub`
302
204
 
303
- Parameters
304
- ----------
305
- cpu : int, default 1
306
- Number of CPUs required for this step. If `@resources` is
307
- also present, the maximum value from all decorators is used.
308
- memory : int, default 4096
309
- Memory size (in MB) required for this step. If
310
- `@resources` is also present, the maximum value from all decorators is
311
- used.
312
- disk : int, default 10240
313
- Disk size (in MB) required for this step. If
314
- `@resources` is also present, the maximum value from all decorators is
315
- used.
316
- image : str, optional, default None
317
- Docker image to use when launching on Kubernetes. If not specified, and
318
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
319
- not, a default Docker image mapping to the current version of Python is used.
320
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
321
- If given, the imagePullPolicy to be applied to the Docker image of the step.
322
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
323
- Kubernetes service account to use when launching pod in Kubernetes.
324
- secrets : List[str], optional, default None
325
- Kubernetes secrets to use when launching pod in Kubernetes. These
326
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
327
- in Metaflow configuration.
328
- node_selector: Union[Dict[str,str], str], optional, default None
329
- Kubernetes node selector(s) to apply to the pod running the task.
330
- Can be passed in as a comma separated string of values e.g.
331
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
332
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
333
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
334
- Kubernetes namespace to use when launching pod in Kubernetes.
335
- gpu : int, optional, default None
336
- Number of GPUs required for this step. A value of zero implies that
337
- the scheduled node should not have GPUs.
338
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
339
- The vendor of the GPUs to be used for this step.
340
- tolerations : List[str], default []
341
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
342
- Kubernetes tolerations to use when launching pod in Kubernetes.
343
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
344
- Kubernetes labels to use when launching pod in Kubernetes.
345
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
346
- Kubernetes annotations to use when launching pod in Kubernetes.
347
- use_tmpfs : bool, default False
348
- This enables an explicit tmpfs mount for this step.
349
- tmpfs_tempdir : bool, default True
350
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
351
- tmpfs_size : int, optional, default: None
352
- The value for the size (in MiB) of the tmpfs mount for this step.
353
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
354
- memory allocated for this step.
355
- tmpfs_path : str, optional, default /metaflow_temp
356
- Path to tmpfs mount for this step.
357
- persistent_volume_claims : Dict[str, str], optional, default None
358
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
359
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
360
- shared_memory: int, optional
361
- Shared memory size (in MiB) required for this step
362
- port: int, optional
363
- Port number to specify in the Kubernetes job object
364
- compute_pool : str, optional, default None
365
- Compute pool to be used for for this step.
366
- If not specified, any accessible compute pool within the perimeter is used.
367
- hostname_resolution_timeout: int, default 10 * 60
368
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
369
- Only applicable when @parallel is used.
370
- qos: str, default: Burstable
371
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
205
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
206
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
207
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
372
208
 
373
- security_context: Dict[str, Any], optional, default None
374
- Container security context. Applies to the task container. Allows the following keys:
375
- - privileged: bool, optional, default None
376
- - allow_privilege_escalation: bool, optional, default None
377
- - run_as_user: int, optional, default None
378
- - run_as_group: int, optional, default None
379
- - run_as_non_root: bool, optional, default None
209
+ temp_dir_root : str, default: None
210
+ The root directory under which `current.model.loaded` will store loaded models
380
211
  """
381
212
  ...
382
213
 
@@ -463,14 +294,76 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
463
294
  ...
464
295
 
465
296
  @typing.overload
466
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
297
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
467
298
  """
468
- Specifies the resources needed when executing this step.
469
-
470
- Use `@resources` to specify the resource requirements
471
- independently of the specific compute layer (`@batch`, `@kubernetes`).
299
+ Decorator prototype for all step decorators. This function gets specialized
300
+ and imported for all decorators types by _import_plugin_decorators().
301
+ """
302
+ ...
303
+
304
+ @typing.overload
305
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
306
+ ...
307
+
308
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
309
+ """
310
+ Decorator prototype for all step decorators. This function gets specialized
311
+ and imported for all decorators types by _import_plugin_decorators().
312
+ """
313
+ ...
314
+
315
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
316
+ """
317
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
472
318
 
473
- You can choose the compute layer on the command line by executing e.g.
319
+ User code call
320
+ --------------
321
+ @ollama(
322
+ models=[...],
323
+ ...
324
+ )
325
+
326
+ Valid backend options
327
+ ---------------------
328
+ - 'local': Run as a separate process on the local task machine.
329
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
330
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
331
+
332
+ Valid model options
333
+ -------------------
334
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
335
+
336
+
337
+ Parameters
338
+ ----------
339
+ models: list[str]
340
+ List of Ollama containers running models in sidecars.
341
+ backend: str
342
+ Determines where and how to run the Ollama process.
343
+ force_pull: bool
344
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
345
+ cache_update_policy: str
346
+ Cache update policy: "auto", "force", or "never".
347
+ force_cache_update: bool
348
+ Simple override for "force" cache update policy.
349
+ debug: bool
350
+ Whether to turn on verbose debugging logs.
351
+ circuit_breaker_config: dict
352
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
353
+ timeout_config: dict
354
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
355
+ """
356
+ ...
357
+
358
+ @typing.overload
359
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
360
+ """
361
+ Specifies the resources needed when executing this step.
362
+
363
+ Use `@resources` to specify the resource requirements
364
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
365
+
366
+ You can choose the compute layer on the command line by executing e.g.
474
367
  ```
475
368
  python myflow.py run --with batch
476
369
  ```
@@ -541,168 +434,124 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
541
434
  """
542
435
  ...
543
436
 
544
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
437
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
545
438
  """
546
- Specifies that this step is used to deploy an instance of the app.
547
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
439
+ Specifies that this step should execute on DGX cloud.
548
440
 
549
441
 
550
442
  Parameters
551
443
  ----------
552
- app_port : int
444
+ gpu : int
553
445
  Number of GPUs to use.
554
- app_name : str
555
- Name of the app to deploy.
446
+ gpu_type : str
447
+ Type of Nvidia GPU to use.
556
448
  """
557
449
  ...
558
450
 
559
451
  @typing.overload
560
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
452
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
561
453
  """
562
- Specifies the number of times the task corresponding
563
- to a step needs to be retried.
564
-
565
- This decorator is useful for handling transient errors, such as networking issues.
566
- If your task contains operations that can't be retried safely, e.g. database updates,
567
- it is advisable to annotate it with `@retry(times=0)`.
568
-
569
- This can be used in conjunction with the `@catch` decorator. The `@catch`
570
- decorator will execute a no-op task after all retries have been exhausted,
571
- ensuring that the flow execution can continue.
572
-
573
-
574
- Parameters
575
- ----------
576
- times : int, default 3
577
- Number of times to retry this task.
578
- minutes_between_retries : int, default 2
579
- Number of minutes between retries.
454
+ Internal decorator to support Fast bakery
580
455
  """
581
456
  ...
582
457
 
583
458
  @typing.overload
584
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
585
- ...
586
-
587
- @typing.overload
588
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
459
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
589
460
  ...
590
461
 
591
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
462
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
592
463
  """
593
- Specifies the number of times the task corresponding
594
- to a step needs to be retried.
595
-
596
- This decorator is useful for handling transient errors, such as networking issues.
597
- If your task contains operations that can't be retried safely, e.g. database updates,
598
- it is advisable to annotate it with `@retry(times=0)`.
599
-
600
- This can be used in conjunction with the `@catch` decorator. The `@catch`
601
- decorator will execute a no-op task after all retries have been exhausted,
602
- ensuring that the flow execution can continue.
603
-
604
-
605
- Parameters
606
- ----------
607
- times : int, default 3
608
- Number of times to retry this task.
609
- minutes_between_retries : int, default 2
610
- Number of minutes between retries.
464
+ Internal decorator to support Fast bakery
611
465
  """
612
466
  ...
613
467
 
614
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
468
+ @typing.overload
469
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
615
470
  """
616
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
471
+ Specifies a timeout for your step.
617
472
 
618
- User code call
619
- --------------
620
- @ollama(
621
- models=[...],
622
- ...
623
- )
473
+ This decorator is useful if this step may hang indefinitely.
624
474
 
625
- Valid backend options
626
- ---------------------
627
- - 'local': Run as a separate process on the local task machine.
628
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
629
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
475
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
476
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
477
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
630
478
 
631
- Valid model options
632
- -------------------
633
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
479
+ Note that all the values specified in parameters are added together so if you specify
480
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
634
481
 
635
482
 
636
483
  Parameters
637
484
  ----------
638
- models: list[str]
639
- List of Ollama containers running models in sidecars.
640
- backend: str
641
- Determines where and how to run the Ollama process.
642
- force_pull: bool
643
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
644
- cache_update_policy: str
645
- Cache update policy: "auto", "force", or "never".
646
- force_cache_update: bool
647
- Simple override for "force" cache update policy.
648
- debug: bool
649
- Whether to turn on verbose debugging logs.
650
- circuit_breaker_config: dict
651
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
652
- timeout_config: dict
653
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
485
+ seconds : int, default 0
486
+ Number of seconds to wait prior to timing out.
487
+ minutes : int, default 0
488
+ Number of minutes to wait prior to timing out.
489
+ hours : int, default 0
490
+ Number of hours to wait prior to timing out.
654
491
  """
655
492
  ...
656
493
 
657
494
  @typing.overload
658
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
659
- """
660
- Internal decorator to support Fast bakery
661
- """
495
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
662
496
  ...
663
497
 
664
498
  @typing.overload
665
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
499
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
666
500
  ...
667
501
 
668
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
502
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
669
503
  """
670
- Internal decorator to support Fast bakery
504
+ Specifies a timeout for your step.
505
+
506
+ This decorator is useful if this step may hang indefinitely.
507
+
508
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
509
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
510
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
511
+
512
+ Note that all the values specified in parameters are added together so if you specify
513
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
514
+
515
+
516
+ Parameters
517
+ ----------
518
+ seconds : int, default 0
519
+ Number of seconds to wait prior to timing out.
520
+ minutes : int, default 0
521
+ Number of minutes to wait prior to timing out.
522
+ hours : int, default 0
523
+ Number of hours to wait prior to timing out.
671
524
  """
672
525
  ...
673
526
 
674
- @typing.overload
675
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
527
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
676
528
  """
677
- Specifies secrets to be retrieved and injected as environment variables prior to
678
- the execution of a step.
529
+ Specifies that this step is used to deploy an instance of the app.
530
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
679
531
 
680
532
 
681
533
  Parameters
682
534
  ----------
683
- sources : List[Union[str, Dict[str, Any]]], default: []
684
- List of secret specs, defining how the secrets are to be retrieved
535
+ app_port : int
536
+ Number of GPUs to use.
537
+ app_name : str
538
+ Name of the app to deploy.
685
539
  """
686
540
  ...
687
541
 
688
- @typing.overload
689
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
690
- ...
691
-
692
- @typing.overload
693
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
694
- ...
695
-
696
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
542
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
697
543
  """
698
- Specifies secrets to be retrieved and injected as environment variables prior to
699
- the execution of a step.
544
+ Specifies that this step should execute on DGX cloud.
700
545
 
701
546
 
702
547
  Parameters
703
548
  ----------
704
- sources : List[Union[str, Dict[str, Any]]], default: []
705
- List of secret specs, defining how the secrets are to be retrieved
549
+ gpu : int
550
+ Number of GPUs to use.
551
+ gpu_type : str
552
+ Type of Nvidia GPU to use.
553
+ queue_timeout : int
554
+ Time to keep the job in NVCF's queue.
706
555
  """
707
556
  ...
708
557
 
@@ -758,218 +607,691 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
758
607
  ...
759
608
 
760
609
  @typing.overload
761
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
610
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
762
611
  """
763
- Specifies the PyPI packages for the step.
612
+ Creates a human-readable report, a Metaflow Card, after this step completes.
764
613
 
765
- Information in this decorator will augment any
766
- attributes set in the `@pyi_base` flow-level decorator. Hence,
767
- you can use `@pypi_base` to set packages required by all
768
- steps and use `@pypi` to specify step-specific overrides.
614
+ Note that you may add multiple `@card` decorators in a step with different parameters.
769
615
 
770
616
 
771
617
  Parameters
772
618
  ----------
773
- packages : Dict[str, str], default: {}
774
- Packages to use for this step. The key is the name of the package
775
- and the value is the version to use.
776
- python : str, optional, default: None
777
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
778
- that the version used will correspond to the version of the Python interpreter used to start the run.
619
+ type : str, default 'default'
620
+ Card type.
621
+ id : str, optional, default None
622
+ If multiple cards are present, use this id to identify this card.
623
+ options : Dict[str, Any], default {}
624
+ Options passed to the card. The contents depend on the card type.
625
+ timeout : int, default 45
626
+ Interrupt reporting if it takes more than this many seconds.
779
627
  """
780
628
  ...
781
629
 
782
630
  @typing.overload
783
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
631
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
784
632
  ...
785
633
 
786
634
  @typing.overload
787
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
635
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
788
636
  ...
789
637
 
790
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
638
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
791
639
  """
792
- Specifies the PyPI packages for the step.
640
+ Creates a human-readable report, a Metaflow Card, after this step completes.
793
641
 
794
- Information in this decorator will augment any
795
- attributes set in the `@pyi_base` flow-level decorator. Hence,
796
- you can use `@pypi_base` to set packages required by all
797
- steps and use `@pypi` to specify step-specific overrides.
642
+ Note that you may add multiple `@card` decorators in a step with different parameters.
798
643
 
799
644
 
800
645
  Parameters
801
646
  ----------
802
- packages : Dict[str, str], default: {}
803
- Packages to use for this step. The key is the name of the package
804
- and the value is the version to use.
805
- python : str, optional, default: None
806
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
807
- that the version used will correspond to the version of the Python interpreter used to start the run.
647
+ type : str, default 'default'
648
+ Card type.
649
+ id : str, optional, default None
650
+ If multiple cards are present, use this id to identify this card.
651
+ options : Dict[str, Any], default {}
652
+ Options passed to the card. The contents depend on the card type.
653
+ timeout : int, default 45
654
+ Interrupt reporting if it takes more than this many seconds.
808
655
  """
809
656
  ...
810
657
 
811
658
  @typing.overload
812
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
659
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
813
660
  """
814
- Specifies environment variables to be set prior to the execution of a step.
661
+ Specifies the Conda environment for the step.
662
+
663
+ Information in this decorator will augment any
664
+ attributes set in the `@conda_base` flow-level decorator. Hence,
665
+ you can use `@conda_base` to set packages required by all
666
+ steps and use `@conda` to specify step-specific overrides.
815
667
 
816
668
 
817
669
  Parameters
818
670
  ----------
819
- vars : Dict[str, str], default {}
820
- Dictionary of environment variables to set.
671
+ packages : Dict[str, str], default {}
672
+ Packages to use for this step. The key is the name of the package
673
+ and the value is the version to use.
674
+ libraries : Dict[str, str], default {}
675
+ Supported for backward compatibility. When used with packages, packages will take precedence.
676
+ python : str, optional, default None
677
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
678
+ that the version used will correspond to the version of the Python interpreter used to start the run.
679
+ disabled : bool, default False
680
+ If set to True, disables @conda.
821
681
  """
822
682
  ...
823
683
 
824
684
  @typing.overload
825
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
685
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
826
686
  ...
827
687
 
828
688
  @typing.overload
829
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
689
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
830
690
  ...
831
691
 
832
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
692
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
833
693
  """
834
- Specifies environment variables to be set prior to the execution of a step.
694
+ Specifies the Conda environment for the step.
695
+
696
+ Information in this decorator will augment any
697
+ attributes set in the `@conda_base` flow-level decorator. Hence,
698
+ you can use `@conda_base` to set packages required by all
699
+ steps and use `@conda` to specify step-specific overrides.
835
700
 
836
701
 
837
702
  Parameters
838
703
  ----------
839
- vars : Dict[str, str], default {}
840
- Dictionary of environment variables to set.
704
+ packages : Dict[str, str], default {}
705
+ Packages to use for this step. The key is the name of the package
706
+ and the value is the version to use.
707
+ libraries : Dict[str, str], default {}
708
+ Supported for backward compatibility. When used with packages, packages will take precedence.
709
+ python : str, optional, default None
710
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
711
+ that the version used will correspond to the version of the Python interpreter used to start the run.
712
+ disabled : bool, default False
713
+ If set to True, disables @conda.
841
714
  """
842
715
  ...
843
716
 
844
- @typing.overload
845
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
717
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
846
718
  """
847
- Enables loading / saving of models within a step.
848
-
719
+ Specifies that this step should execute on Kubernetes.
849
720
 
850
721
 
851
722
  Parameters
852
723
  ----------
853
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
854
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
855
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
856
- - `current.checkpoint`
857
- - `current.model`
858
- - `current.huggingface_hub`
859
-
860
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
861
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
862
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
724
+ cpu : int, default 1
725
+ Number of CPUs required for this step. If `@resources` is
726
+ also present, the maximum value from all decorators is used.
727
+ memory : int, default 4096
728
+ Memory size (in MB) required for this step. If
729
+ `@resources` is also present, the maximum value from all decorators is
730
+ used.
731
+ disk : int, default 10240
732
+ Disk size (in MB) required for this step. If
733
+ `@resources` is also present, the maximum value from all decorators is
734
+ used.
735
+ image : str, optional, default None
736
+ Docker image to use when launching on Kubernetes. If not specified, and
737
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
738
+ not, a default Docker image mapping to the current version of Python is used.
739
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
740
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
741
+ image_pull_secrets: List[str], default []
742
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
743
+ Kubernetes image pull secrets to use when pulling container images
744
+ in Kubernetes.
745
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
746
+ Kubernetes service account to use when launching pod in Kubernetes.
747
+ secrets : List[str], optional, default None
748
+ Kubernetes secrets to use when launching pod in Kubernetes. These
749
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
750
+ in Metaflow configuration.
751
+ node_selector: Union[Dict[str,str], str], optional, default None
752
+ Kubernetes node selector(s) to apply to the pod running the task.
753
+ Can be passed in as a comma separated string of values e.g.
754
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
755
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
756
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
757
+ Kubernetes namespace to use when launching pod in Kubernetes.
758
+ gpu : int, optional, default None
759
+ Number of GPUs required for this step. A value of zero implies that
760
+ the scheduled node should not have GPUs.
761
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
762
+ The vendor of the GPUs to be used for this step.
763
+ tolerations : List[str], default []
764
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
765
+ Kubernetes tolerations to use when launching pod in Kubernetes.
766
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
767
+ Kubernetes labels to use when launching pod in Kubernetes.
768
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
769
+ Kubernetes annotations to use when launching pod in Kubernetes.
770
+ use_tmpfs : bool, default False
771
+ This enables an explicit tmpfs mount for this step.
772
+ tmpfs_tempdir : bool, default True
773
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
774
+ tmpfs_size : int, optional, default: None
775
+ The value for the size (in MiB) of the tmpfs mount for this step.
776
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
777
+ memory allocated for this step.
778
+ tmpfs_path : str, optional, default /metaflow_temp
779
+ Path to tmpfs mount for this step.
780
+ persistent_volume_claims : Dict[str, str], optional, default None
781
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
782
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
783
+ shared_memory: int, optional
784
+ Shared memory size (in MiB) required for this step
785
+ port: int, optional
786
+ Port number to specify in the Kubernetes job object
787
+ compute_pool : str, optional, default None
788
+ Compute pool to be used for for this step.
789
+ If not specified, any accessible compute pool within the perimeter is used.
790
+ hostname_resolution_timeout: int, default 10 * 60
791
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
792
+ Only applicable when @parallel is used.
793
+ qos: str, default: Burstable
794
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
863
795
 
864
- temp_dir_root : str, default: None
865
- The root directory under which `current.model.loaded` will store loaded models
796
+ security_context: Dict[str, Any], optional, default None
797
+ Container security context. Applies to the task container. Allows the following keys:
798
+ - privileged: bool, optional, default None
799
+ - allow_privilege_escalation: bool, optional, default None
800
+ - run_as_user: int, optional, default None
801
+ - run_as_group: int, optional, default None
802
+ - run_as_non_root: bool, optional, default None
866
803
  """
867
804
  ...
868
805
 
869
- @typing.overload
870
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
871
- ...
872
-
873
- @typing.overload
874
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
875
- ...
876
-
877
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
806
+ def vllm(*, model: str, backend: str, debug: bool, kwargs: typing.Any) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
878
807
  """
879
- Enables loading / saving of models within a step.
808
+ This decorator is used to run vllm APIs as Metaflow task sidecars.
809
+
810
+ User code call
811
+ --------------
812
+ @vllm(
813
+ model="...",
814
+ ...
815
+ )
880
816
 
817
+ Valid backend options
818
+ ---------------------
819
+ - 'local': Run as a separate process on the local task machine.
881
820
 
821
+ Valid model options
822
+ -------------------
823
+ Any HuggingFace model identifier, e.g. 'meta-llama/Llama-3.2-1B'
882
824
 
883
- Parameters
884
- ----------
885
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
886
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
887
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
888
- - `current.checkpoint`
889
- - `current.model`
890
- - `current.huggingface_hub`
825
+ NOTE: vLLM's OpenAI-compatible server serves ONE model per server instance.
826
+ If you need multiple models, you must create multiple @vllm decorators.
891
827
 
892
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
893
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
894
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
895
828
 
896
- temp_dir_root : str, default: None
897
- The root directory under which `current.model.loaded` will store loaded models
829
+ Parameters
830
+ ----------
831
+ model: str
832
+ HuggingFace model identifier to be served by vLLM.
833
+ backend: str
834
+ Determines where and how to run the vLLM process.
835
+ debug: bool
836
+ Whether to turn on verbose debugging logs.
837
+ kwargs : Any
838
+ Any other keyword arguments are passed directly to the vLLM engine.
839
+ This allows for flexible configuration of vLLM server settings.
840
+ For example, `tensor_parallel_size=2`.
898
841
  """
899
842
  ...
900
843
 
901
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
844
+ @typing.overload
845
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
902
846
  """
903
- Specifies that this step should execute on DGX cloud.
847
+ Specifies the PyPI packages for the step.
848
+
849
+ Information in this decorator will augment any
850
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
851
+ you can use `@pypi_base` to set packages required by all
852
+ steps and use `@pypi` to specify step-specific overrides.
904
853
 
905
854
 
906
855
  Parameters
907
856
  ----------
908
- gpu : int
909
- Number of GPUs to use.
910
- gpu_type : str
911
- Type of Nvidia GPU to use.
912
- queue_timeout : int
913
- Time to keep the job in NVCF's queue.
857
+ packages : Dict[str, str], default: {}
858
+ Packages to use for this step. The key is the name of the package
859
+ and the value is the version to use.
860
+ python : str, optional, default: None
861
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
862
+ that the version used will correspond to the version of the Python interpreter used to start the run.
914
863
  """
915
864
  ...
916
865
 
917
866
  @typing.overload
918
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
867
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
868
+ ...
869
+
870
+ @typing.overload
871
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
872
+ ...
873
+
874
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
919
875
  """
920
- Specifies the Conda environment for the step.
876
+ Specifies the PyPI packages for the step.
921
877
 
922
878
  Information in this decorator will augment any
923
- attributes set in the `@conda_base` flow-level decorator. Hence,
924
- you can use `@conda_base` to set packages required by all
925
- steps and use `@conda` to specify step-specific overrides.
879
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
880
+ you can use `@pypi_base` to set packages required by all
881
+ steps and use `@pypi` to specify step-specific overrides.
926
882
 
927
883
 
928
884
  Parameters
929
885
  ----------
930
- packages : Dict[str, str], default {}
886
+ packages : Dict[str, str], default: {}
931
887
  Packages to use for this step. The key is the name of the package
932
888
  and the value is the version to use.
933
- libraries : Dict[str, str], default {}
934
- Supported for backward compatibility. When used with packages, packages will take precedence.
935
- python : str, optional, default None
889
+ python : str, optional, default: None
936
890
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
937
891
  that the version used will correspond to the version of the Python interpreter used to start the run.
938
- disabled : bool, default False
939
- If set to True, disables @conda.
940
892
  """
941
893
  ...
942
894
 
943
- @typing.overload
944
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
945
- ...
946
-
947
- @typing.overload
948
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
949
- ...
950
-
951
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
895
+ @typing.overload
896
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
897
+ """
898
+ Specifies the number of times the task corresponding
899
+ to a step needs to be retried.
900
+
901
+ This decorator is useful for handling transient errors, such as networking issues.
902
+ If your task contains operations that can't be retried safely, e.g. database updates,
903
+ it is advisable to annotate it with `@retry(times=0)`.
904
+
905
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
906
+ decorator will execute a no-op task after all retries have been exhausted,
907
+ ensuring that the flow execution can continue.
908
+
909
+
910
+ Parameters
911
+ ----------
912
+ times : int, default 3
913
+ Number of times to retry this task.
914
+ minutes_between_retries : int, default 2
915
+ Number of minutes between retries.
916
+ """
917
+ ...
918
+
919
+ @typing.overload
920
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
921
+ ...
922
+
923
+ @typing.overload
924
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
925
+ ...
926
+
927
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
928
+ """
929
+ Specifies the number of times the task corresponding
930
+ to a step needs to be retried.
931
+
932
+ This decorator is useful for handling transient errors, such as networking issues.
933
+ If your task contains operations that can't be retried safely, e.g. database updates,
934
+ it is advisable to annotate it with `@retry(times=0)`.
935
+
936
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
937
+ decorator will execute a no-op task after all retries have been exhausted,
938
+ ensuring that the flow execution can continue.
939
+
940
+
941
+ Parameters
942
+ ----------
943
+ times : int, default 3
944
+ Number of times to retry this task.
945
+ minutes_between_retries : int, default 2
946
+ Number of minutes between retries.
947
+ """
948
+ ...
949
+
950
+ @typing.overload
951
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
952
+ """
953
+ Specifies environment variables to be set prior to the execution of a step.
954
+
955
+
956
+ Parameters
957
+ ----------
958
+ vars : Dict[str, str], default {}
959
+ Dictionary of environment variables to set.
960
+ """
961
+ ...
962
+
963
+ @typing.overload
964
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
965
+ ...
966
+
967
+ @typing.overload
968
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
969
+ ...
970
+
971
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
972
+ """
973
+ Specifies environment variables to be set prior to the execution of a step.
974
+
975
+
976
+ Parameters
977
+ ----------
978
+ vars : Dict[str, str], default {}
979
+ Dictionary of environment variables to set.
980
+ """
981
+ ...
982
+
983
+ @typing.overload
984
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
985
+ """
986
+ Specifies secrets to be retrieved and injected as environment variables prior to
987
+ the execution of a step.
988
+
989
+
990
+ Parameters
991
+ ----------
992
+ sources : List[Union[str, Dict[str, Any]]], default: []
993
+ List of secret specs, defining how the secrets are to be retrieved
994
+ """
995
+ ...
996
+
997
+ @typing.overload
998
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
999
+ ...
1000
+
1001
+ @typing.overload
1002
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1003
+ ...
1004
+
1005
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1006
+ """
1007
+ Specifies secrets to be retrieved and injected as environment variables prior to
1008
+ the execution of a step.
1009
+
1010
+
1011
+ Parameters
1012
+ ----------
1013
+ sources : List[Union[str, Dict[str, Any]]], default: []
1014
+ List of secret specs, defining how the secrets are to be retrieved
1015
+ """
1016
+ ...
1017
+
1018
+ @typing.overload
1019
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1020
+ """
1021
+ Specifies the event(s) that this flow depends on.
1022
+
1023
+ ```
1024
+ @trigger(event='foo')
1025
+ ```
1026
+ or
1027
+ ```
1028
+ @trigger(events=['foo', 'bar'])
1029
+ ```
1030
+
1031
+ Additionally, you can specify the parameter mappings
1032
+ to map event payload to Metaflow parameters for the flow.
1033
+ ```
1034
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1035
+ ```
1036
+ or
1037
+ ```
1038
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1039
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1040
+ ```
1041
+
1042
+ 'parameters' can also be a list of strings and tuples like so:
1043
+ ```
1044
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1045
+ ```
1046
+ This is equivalent to:
1047
+ ```
1048
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1049
+ ```
1050
+
1051
+
1052
+ Parameters
1053
+ ----------
1054
+ event : Union[str, Dict[str, Any]], optional, default None
1055
+ Event dependency for this flow.
1056
+ events : List[Union[str, Dict[str, Any]]], default []
1057
+ Events dependency for this flow.
1058
+ options : Dict[str, Any], default {}
1059
+ Backend-specific configuration for tuning eventing behavior.
1060
+ """
1061
+ ...
1062
+
1063
+ @typing.overload
1064
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1065
+ ...
1066
+
1067
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1068
+ """
1069
+ Specifies the event(s) that this flow depends on.
1070
+
1071
+ ```
1072
+ @trigger(event='foo')
1073
+ ```
1074
+ or
1075
+ ```
1076
+ @trigger(events=['foo', 'bar'])
1077
+ ```
1078
+
1079
+ Additionally, you can specify the parameter mappings
1080
+ to map event payload to Metaflow parameters for the flow.
1081
+ ```
1082
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1083
+ ```
1084
+ or
1085
+ ```
1086
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1087
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1088
+ ```
1089
+
1090
+ 'parameters' can also be a list of strings and tuples like so:
1091
+ ```
1092
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1093
+ ```
1094
+ This is equivalent to:
1095
+ ```
1096
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1097
+ ```
1098
+
1099
+
1100
+ Parameters
1101
+ ----------
1102
+ event : Union[str, Dict[str, Any]], optional, default None
1103
+ Event dependency for this flow.
1104
+ events : List[Union[str, Dict[str, Any]]], default []
1105
+ Events dependency for this flow.
1106
+ options : Dict[str, Any], default {}
1107
+ Backend-specific configuration for tuning eventing behavior.
1108
+ """
1109
+ ...
1110
+
1111
+ @typing.overload
1112
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1113
+ """
1114
+ Specifies the flow(s) that this flow depends on.
1115
+
1116
+ ```
1117
+ @trigger_on_finish(flow='FooFlow')
1118
+ ```
1119
+ or
1120
+ ```
1121
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1122
+ ```
1123
+ This decorator respects the @project decorator and triggers the flow
1124
+ when upstream runs within the same namespace complete successfully
1125
+
1126
+ Additionally, you can specify project aware upstream flow dependencies
1127
+ by specifying the fully qualified project_flow_name.
1128
+ ```
1129
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1130
+ ```
1131
+ or
1132
+ ```
1133
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1134
+ ```
1135
+
1136
+ You can also specify just the project or project branch (other values will be
1137
+ inferred from the current project or project branch):
1138
+ ```
1139
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1140
+ ```
1141
+
1142
+ Note that `branch` is typically one of:
1143
+ - `prod`
1144
+ - `user.bob`
1145
+ - `test.my_experiment`
1146
+ - `prod.staging`
1147
+
1148
+
1149
+ Parameters
1150
+ ----------
1151
+ flow : Union[str, Dict[str, str]], optional, default None
1152
+ Upstream flow dependency for this flow.
1153
+ flows : List[Union[str, Dict[str, str]]], default []
1154
+ Upstream flow dependencies for this flow.
1155
+ options : Dict[str, Any], default {}
1156
+ Backend-specific configuration for tuning eventing behavior.
1157
+ """
1158
+ ...
1159
+
1160
+ @typing.overload
1161
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1162
+ ...
1163
+
1164
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1165
+ """
1166
+ Specifies the flow(s) that this flow depends on.
1167
+
1168
+ ```
1169
+ @trigger_on_finish(flow='FooFlow')
1170
+ ```
1171
+ or
1172
+ ```
1173
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1174
+ ```
1175
+ This decorator respects the @project decorator and triggers the flow
1176
+ when upstream runs within the same namespace complete successfully
1177
+
1178
+ Additionally, you can specify project aware upstream flow dependencies
1179
+ by specifying the fully qualified project_flow_name.
1180
+ ```
1181
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1182
+ ```
1183
+ or
1184
+ ```
1185
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1186
+ ```
1187
+
1188
+ You can also specify just the project or project branch (other values will be
1189
+ inferred from the current project or project branch):
1190
+ ```
1191
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1192
+ ```
1193
+
1194
+ Note that `branch` is typically one of:
1195
+ - `prod`
1196
+ - `user.bob`
1197
+ - `test.my_experiment`
1198
+ - `prod.staging`
1199
+
1200
+
1201
+ Parameters
1202
+ ----------
1203
+ flow : Union[str, Dict[str, str]], optional, default None
1204
+ Upstream flow dependency for this flow.
1205
+ flows : List[Union[str, Dict[str, str]]], default []
1206
+ Upstream flow dependencies for this flow.
1207
+ options : Dict[str, Any], default {}
1208
+ Backend-specific configuration for tuning eventing behavior.
1209
+ """
1210
+ ...
1211
+
1212
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1213
+ """
1214
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1215
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1216
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1217
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1218
+ starts only after all sensors finish.
1219
+
1220
+
1221
+ Parameters
1222
+ ----------
1223
+ timeout : int
1224
+ Time, in seconds before the task times out and fails. (Default: 3600)
1225
+ poke_interval : int
1226
+ Time in seconds that the job should wait in between each try. (Default: 60)
1227
+ mode : str
1228
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1229
+ exponential_backoff : bool
1230
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1231
+ pool : str
1232
+ the slot pool this task should run in,
1233
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1234
+ soft_fail : bool
1235
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1236
+ name : str
1237
+ Name of the sensor on Airflow
1238
+ description : str
1239
+ Description of sensor in the Airflow UI
1240
+ bucket_key : Union[str, List[str]]
1241
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1242
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1243
+ bucket_name : str
1244
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1245
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1246
+ wildcard_match : bool
1247
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1248
+ aws_conn_id : str
1249
+ a reference to the s3 connection on Airflow. (Default: None)
1250
+ verify : bool
1251
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1252
+ """
1253
+ ...
1254
+
1255
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
952
1256
  """
953
- Specifies the Conda environment for the step.
954
-
955
- Information in this decorator will augment any
956
- attributes set in the `@conda_base` flow-level decorator. Hence,
957
- you can use `@conda_base` to set packages required by all
958
- steps and use `@conda` to specify step-specific overrides.
1257
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1258
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
959
1259
 
960
1260
 
961
1261
  Parameters
962
1262
  ----------
963
- packages : Dict[str, str], default {}
964
- Packages to use for this step. The key is the name of the package
965
- and the value is the version to use.
966
- libraries : Dict[str, str], default {}
967
- Supported for backward compatibility. When used with packages, packages will take precedence.
968
- python : str, optional, default None
969
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
970
- that the version used will correspond to the version of the Python interpreter used to start the run.
971
- disabled : bool, default False
972
- If set to True, disables @conda.
1263
+ timeout : int
1264
+ Time, in seconds before the task times out and fails. (Default: 3600)
1265
+ poke_interval : int
1266
+ Time in seconds that the job should wait in between each try. (Default: 60)
1267
+ mode : str
1268
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1269
+ exponential_backoff : bool
1270
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1271
+ pool : str
1272
+ the slot pool this task should run in,
1273
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1274
+ soft_fail : bool
1275
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1276
+ name : str
1277
+ Name of the sensor on Airflow
1278
+ description : str
1279
+ Description of sensor in the Airflow UI
1280
+ external_dag_id : str
1281
+ The dag_id that contains the task you want to wait for.
1282
+ external_task_ids : List[str]
1283
+ The list of task_ids that you want to wait for.
1284
+ If None (default value) the sensor waits for the DAG. (Default: None)
1285
+ allowed_states : List[str]
1286
+ Iterable of allowed states, (Default: ['success'])
1287
+ failed_states : List[str]
1288
+ Iterable of failed or dis-allowed states. (Default: None)
1289
+ execution_delta : datetime.timedelta
1290
+ time difference with the previous execution to look at,
1291
+ the default is the same logical date as the current task or DAG. (Default: None)
1292
+ check_existence: bool
1293
+ Set to True to check if the external task exists or check if
1294
+ the DAG to wait for exists. (Default: True)
973
1295
  """
974
1296
  ...
975
1297
 
@@ -1072,190 +1394,18 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1072
1394
  Parameters:
1073
1395
  ----------
1074
1396
 
1075
- type: str
1076
- The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1077
-
1078
- config: dict or Callable
1079
- Dictionary of configuration options for the datastore. The following keys are required:
1080
- - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1081
- - example: 's3://bucket-name/path/to/root'
1082
- - example: 'gs://bucket-name/path/to/root'
1083
- - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1084
- - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1085
- - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1086
- - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1087
- """
1088
- ...
1089
-
1090
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1091
- """
1092
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1093
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1094
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1095
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1096
- starts only after all sensors finish.
1097
-
1098
-
1099
- Parameters
1100
- ----------
1101
- timeout : int
1102
- Time, in seconds before the task times out and fails. (Default: 3600)
1103
- poke_interval : int
1104
- Time in seconds that the job should wait in between each try. (Default: 60)
1105
- mode : str
1106
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1107
- exponential_backoff : bool
1108
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1109
- pool : str
1110
- the slot pool this task should run in,
1111
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1112
- soft_fail : bool
1113
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1114
- name : str
1115
- Name of the sensor on Airflow
1116
- description : str
1117
- Description of sensor in the Airflow UI
1118
- bucket_key : Union[str, List[str]]
1119
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1120
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1121
- bucket_name : str
1122
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1123
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1124
- wildcard_match : bool
1125
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1126
- aws_conn_id : str
1127
- a reference to the s3 connection on Airflow. (Default: None)
1128
- verify : bool
1129
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1130
- """
1131
- ...
1132
-
1133
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1134
- """
1135
- Specifies what flows belong to the same project.
1136
-
1137
- A project-specific namespace is created for all flows that
1138
- use the same `@project(name)`.
1139
-
1140
-
1141
- Parameters
1142
- ----------
1143
- name : str
1144
- Project name. Make sure that the name is unique amongst all
1145
- projects that use the same production scheduler. The name may
1146
- contain only lowercase alphanumeric characters and underscores.
1147
-
1148
- branch : Optional[str], default None
1149
- The branch to use. If not specified, the branch is set to
1150
- `user.<username>` unless `production` is set to `True`. This can
1151
- also be set on the command line using `--branch` as a top-level option.
1152
- It is an error to specify `branch` in the decorator and on the command line.
1153
-
1154
- production : bool, default False
1155
- Whether or not the branch is the production branch. This can also be set on the
1156
- command line using `--production` as a top-level option. It is an error to specify
1157
- `production` in the decorator and on the command line.
1158
- The project branch name will be:
1159
- - if `branch` is specified:
1160
- - if `production` is True: `prod.<branch>`
1161
- - if `production` is False: `test.<branch>`
1162
- - if `branch` is not specified:
1163
- - if `production` is True: `prod`
1164
- - if `production` is False: `user.<username>`
1165
- """
1166
- ...
1167
-
1168
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1169
- """
1170
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1171
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1172
-
1173
-
1174
- Parameters
1175
- ----------
1176
- timeout : int
1177
- Time, in seconds before the task times out and fails. (Default: 3600)
1178
- poke_interval : int
1179
- Time in seconds that the job should wait in between each try. (Default: 60)
1180
- mode : str
1181
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1182
- exponential_backoff : bool
1183
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1184
- pool : str
1185
- the slot pool this task should run in,
1186
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1187
- soft_fail : bool
1188
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1189
- name : str
1190
- Name of the sensor on Airflow
1191
- description : str
1192
- Description of sensor in the Airflow UI
1193
- external_dag_id : str
1194
- The dag_id that contains the task you want to wait for.
1195
- external_task_ids : List[str]
1196
- The list of task_ids that you want to wait for.
1197
- If None (default value) the sensor waits for the DAG. (Default: None)
1198
- allowed_states : List[str]
1199
- Iterable of allowed states, (Default: ['success'])
1200
- failed_states : List[str]
1201
- Iterable of failed or dis-allowed states. (Default: None)
1202
- execution_delta : datetime.timedelta
1203
- time difference with the previous execution to look at,
1204
- the default is the same logical date as the current task or DAG. (Default: None)
1205
- check_existence: bool
1206
- Set to True to check if the external task exists or check if
1207
- the DAG to wait for exists. (Default: True)
1208
- """
1209
- ...
1210
-
1211
- @typing.overload
1212
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1213
- """
1214
- Specifies the Conda environment for all steps of the flow.
1215
-
1216
- Use `@conda_base` to set common libraries required by all
1217
- steps and use `@conda` to specify step-specific additions.
1218
-
1219
-
1220
- Parameters
1221
- ----------
1222
- packages : Dict[str, str], default {}
1223
- Packages to use for this flow. The key is the name of the package
1224
- and the value is the version to use.
1225
- libraries : Dict[str, str], default {}
1226
- Supported for backward compatibility. When used with packages, packages will take precedence.
1227
- python : str, optional, default None
1228
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1229
- that the version used will correspond to the version of the Python interpreter used to start the run.
1230
- disabled : bool, default False
1231
- If set to True, disables Conda.
1232
- """
1233
- ...
1234
-
1235
- @typing.overload
1236
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1237
- ...
1238
-
1239
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1240
- """
1241
- Specifies the Conda environment for all steps of the flow.
1242
-
1243
- Use `@conda_base` to set common libraries required by all
1244
- steps and use `@conda` to specify step-specific additions.
1245
-
1246
-
1247
- Parameters
1248
- ----------
1249
- packages : Dict[str, str], default {}
1250
- Packages to use for this flow. The key is the name of the package
1251
- and the value is the version to use.
1252
- libraries : Dict[str, str], default {}
1253
- Supported for backward compatibility. When used with packages, packages will take precedence.
1254
- python : str, optional, default None
1255
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1256
- that the version used will correspond to the version of the Python interpreter used to start the run.
1257
- disabled : bool, default False
1258
- If set to True, disables Conda.
1397
+ type: str
1398
+ The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1399
+
1400
+ config: dict or Callable
1401
+ Dictionary of configuration options for the datastore. The following keys are required:
1402
+ - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1403
+ - example: 's3://bucket-name/path/to/root'
1404
+ - example: 'gs://bucket-name/path/to/root'
1405
+ - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1406
+ - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1407
+ - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1408
+ - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1259
1409
  """
1260
1410
  ...
1261
1411
 
@@ -1311,237 +1461,129 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1311
1461
  ...
1312
1462
 
1313
1463
  @typing.overload
1314
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1464
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1315
1465
  """
1316
- Specifies the flow(s) that this flow depends on.
1317
-
1318
- ```
1319
- @trigger_on_finish(flow='FooFlow')
1320
- ```
1321
- or
1322
- ```
1323
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1324
- ```
1325
- This decorator respects the @project decorator and triggers the flow
1326
- when upstream runs within the same namespace complete successfully
1327
-
1328
- Additionally, you can specify project aware upstream flow dependencies
1329
- by specifying the fully qualified project_flow_name.
1330
- ```
1331
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1332
- ```
1333
- or
1334
- ```
1335
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1336
- ```
1337
-
1338
- You can also specify just the project or project branch (other values will be
1339
- inferred from the current project or project branch):
1340
- ```
1341
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1342
- ```
1343
-
1344
- Note that `branch` is typically one of:
1345
- - `prod`
1346
- - `user.bob`
1347
- - `test.my_experiment`
1348
- - `prod.staging`
1466
+ Specifies the PyPI packages for all steps of the flow.
1349
1467
 
1468
+ Use `@pypi_base` to set common packages required by all
1469
+ steps and use `@pypi` to specify step-specific overrides.
1350
1470
 
1351
1471
  Parameters
1352
1472
  ----------
1353
- flow : Union[str, Dict[str, str]], optional, default None
1354
- Upstream flow dependency for this flow.
1355
- flows : List[Union[str, Dict[str, str]]], default []
1356
- Upstream flow dependencies for this flow.
1357
- options : Dict[str, Any], default {}
1358
- Backend-specific configuration for tuning eventing behavior.
1473
+ packages : Dict[str, str], default: {}
1474
+ Packages to use for this flow. The key is the name of the package
1475
+ and the value is the version to use.
1476
+ python : str, optional, default: None
1477
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1478
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1359
1479
  """
1360
1480
  ...
1361
1481
 
1362
1482
  @typing.overload
1363
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1483
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1364
1484
  ...
1365
1485
 
1366
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1486
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1367
1487
  """
1368
- Specifies the flow(s) that this flow depends on.
1369
-
1370
- ```
1371
- @trigger_on_finish(flow='FooFlow')
1372
- ```
1373
- or
1374
- ```
1375
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1376
- ```
1377
- This decorator respects the @project decorator and triggers the flow
1378
- when upstream runs within the same namespace complete successfully
1379
-
1380
- Additionally, you can specify project aware upstream flow dependencies
1381
- by specifying the fully qualified project_flow_name.
1382
- ```
1383
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1384
- ```
1385
- or
1386
- ```
1387
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1388
- ```
1389
-
1390
- You can also specify just the project or project branch (other values will be
1391
- inferred from the current project or project branch):
1392
- ```
1393
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1394
- ```
1395
-
1396
- Note that `branch` is typically one of:
1397
- - `prod`
1398
- - `user.bob`
1399
- - `test.my_experiment`
1400
- - `prod.staging`
1488
+ Specifies the PyPI packages for all steps of the flow.
1401
1489
 
1490
+ Use `@pypi_base` to set common packages required by all
1491
+ steps and use `@pypi` to specify step-specific overrides.
1402
1492
 
1403
1493
  Parameters
1404
1494
  ----------
1405
- flow : Union[str, Dict[str, str]], optional, default None
1406
- Upstream flow dependency for this flow.
1407
- flows : List[Union[str, Dict[str, str]]], default []
1408
- Upstream flow dependencies for this flow.
1409
- options : Dict[str, Any], default {}
1410
- Backend-specific configuration for tuning eventing behavior.
1495
+ packages : Dict[str, str], default: {}
1496
+ Packages to use for this flow. The key is the name of the package
1497
+ and the value is the version to use.
1498
+ python : str, optional, default: None
1499
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1500
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1411
1501
  """
1412
1502
  ...
1413
1503
 
1414
- @typing.overload
1415
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1504
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1416
1505
  """
1417
- Specifies the event(s) that this flow depends on.
1418
-
1419
- ```
1420
- @trigger(event='foo')
1421
- ```
1422
- or
1423
- ```
1424
- @trigger(events=['foo', 'bar'])
1425
- ```
1426
-
1427
- Additionally, you can specify the parameter mappings
1428
- to map event payload to Metaflow parameters for the flow.
1429
- ```
1430
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1431
- ```
1432
- or
1433
- ```
1434
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1435
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1436
- ```
1506
+ Specifies what flows belong to the same project.
1437
1507
 
1438
- 'parameters' can also be a list of strings and tuples like so:
1439
- ```
1440
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1441
- ```
1442
- This is equivalent to:
1443
- ```
1444
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1445
- ```
1508
+ A project-specific namespace is created for all flows that
1509
+ use the same `@project(name)`.
1446
1510
 
1447
1511
 
1448
1512
  Parameters
1449
1513
  ----------
1450
- event : Union[str, Dict[str, Any]], optional, default None
1451
- Event dependency for this flow.
1452
- events : List[Union[str, Dict[str, Any]]], default []
1453
- Events dependency for this flow.
1454
- options : Dict[str, Any], default {}
1455
- Backend-specific configuration for tuning eventing behavior.
1456
- """
1457
- ...
1458
-
1459
- @typing.overload
1460
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1461
- ...
1462
-
1463
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1464
- """
1465
- Specifies the event(s) that this flow depends on.
1466
-
1467
- ```
1468
- @trigger(event='foo')
1469
- ```
1470
- or
1471
- ```
1472
- @trigger(events=['foo', 'bar'])
1473
- ```
1474
-
1475
- Additionally, you can specify the parameter mappings
1476
- to map event payload to Metaflow parameters for the flow.
1477
- ```
1478
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1479
- ```
1480
- or
1481
- ```
1482
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1483
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1484
- ```
1485
-
1486
- 'parameters' can also be a list of strings and tuples like so:
1487
- ```
1488
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1489
- ```
1490
- This is equivalent to:
1491
- ```
1492
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1493
- ```
1514
+ name : str
1515
+ Project name. Make sure that the name is unique amongst all
1516
+ projects that use the same production scheduler. The name may
1517
+ contain only lowercase alphanumeric characters and underscores.
1494
1518
 
1519
+ branch : Optional[str], default None
1520
+ The branch to use. If not specified, the branch is set to
1521
+ `user.<username>` unless `production` is set to `True`. This can
1522
+ also be set on the command line using `--branch` as a top-level option.
1523
+ It is an error to specify `branch` in the decorator and on the command line.
1495
1524
 
1496
- Parameters
1497
- ----------
1498
- event : Union[str, Dict[str, Any]], optional, default None
1499
- Event dependency for this flow.
1500
- events : List[Union[str, Dict[str, Any]]], default []
1501
- Events dependency for this flow.
1502
- options : Dict[str, Any], default {}
1503
- Backend-specific configuration for tuning eventing behavior.
1525
+ production : bool, default False
1526
+ Whether or not the branch is the production branch. This can also be set on the
1527
+ command line using `--production` as a top-level option. It is an error to specify
1528
+ `production` in the decorator and on the command line.
1529
+ The project branch name will be:
1530
+ - if `branch` is specified:
1531
+ - if `production` is True: `prod.<branch>`
1532
+ - if `production` is False: `test.<branch>`
1533
+ - if `branch` is not specified:
1534
+ - if `production` is True: `prod`
1535
+ - if `production` is False: `user.<username>`
1504
1536
  """
1505
1537
  ...
1506
1538
 
1507
1539
  @typing.overload
1508
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1540
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1509
1541
  """
1510
- Specifies the PyPI packages for all steps of the flow.
1542
+ Specifies the Conda environment for all steps of the flow.
1543
+
1544
+ Use `@conda_base` to set common libraries required by all
1545
+ steps and use `@conda` to specify step-specific additions.
1511
1546
 
1512
- Use `@pypi_base` to set common packages required by all
1513
- steps and use `@pypi` to specify step-specific overrides.
1514
1547
 
1515
1548
  Parameters
1516
1549
  ----------
1517
- packages : Dict[str, str], default: {}
1550
+ packages : Dict[str, str], default {}
1518
1551
  Packages to use for this flow. The key is the name of the package
1519
1552
  and the value is the version to use.
1520
- python : str, optional, default: None
1553
+ libraries : Dict[str, str], default {}
1554
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1555
+ python : str, optional, default None
1521
1556
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1522
1557
  that the version used will correspond to the version of the Python interpreter used to start the run.
1558
+ disabled : bool, default False
1559
+ If set to True, disables Conda.
1523
1560
  """
1524
1561
  ...
1525
1562
 
1526
1563
  @typing.overload
1527
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1564
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1528
1565
  ...
1529
1566
 
1530
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1567
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1531
1568
  """
1532
- Specifies the PyPI packages for all steps of the flow.
1569
+ Specifies the Conda environment for all steps of the flow.
1570
+
1571
+ Use `@conda_base` to set common libraries required by all
1572
+ steps and use `@conda` to specify step-specific additions.
1533
1573
 
1534
- Use `@pypi_base` to set common packages required by all
1535
- steps and use `@pypi` to specify step-specific overrides.
1536
1574
 
1537
1575
  Parameters
1538
1576
  ----------
1539
- packages : Dict[str, str], default: {}
1577
+ packages : Dict[str, str], default {}
1540
1578
  Packages to use for this flow. The key is the name of the package
1541
1579
  and the value is the version to use.
1542
- python : str, optional, default: None
1580
+ libraries : Dict[str, str], default {}
1581
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1582
+ python : str, optional, default None
1543
1583
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1544
1584
  that the version used will correspond to the version of the Python interpreter used to start the run.
1585
+ disabled : bool, default False
1586
+ If set to True, disables Conda.
1545
1587
  """
1546
1588
  ...
1547
1589