ob-metaflow-stubs 6.0.3.180rc2__py2.py3-none-any.whl → 6.0.3.180rc4__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. metaflow-stubs/__init__.pyi +697 -697
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +4 -4
  8. metaflow-stubs/client/filecache.pyi +1 -1
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +95 -95
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +1 -1
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +1 -1
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +1 -1
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +1 -1
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +1 -1
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +1 -1
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +1 -1
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  87. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  88. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  89. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
  90. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  91. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
  92. metaflow-stubs/multicore_utils.pyi +1 -1
  93. metaflow-stubs/ob_internal.pyi +1 -1
  94. metaflow-stubs/parameters.pyi +3 -3
  95. metaflow-stubs/plugins/__init__.pyi +10 -10
  96. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  97. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  98. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  99. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  100. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  101. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  102. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  103. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  104. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  105. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  106. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  107. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  109. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  110. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  111. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  112. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  113. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  114. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  115. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  116. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  117. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  118. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  119. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  120. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  121. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  122. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  123. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  124. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  125. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  126. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  127. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  128. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  129. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  130. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  131. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  132. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  133. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  134. metaflow-stubs/plugins/cards/card_client.pyi +1 -1
  135. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  136. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  137. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  138. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  139. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  140. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  141. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  142. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  143. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  144. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  145. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  146. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  147. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  148. metaflow-stubs/plugins/catch_decorator.pyi +1 -1
  149. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  150. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  151. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  152. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  153. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  154. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  155. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  156. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  157. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  158. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  159. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  160. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  161. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  162. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  163. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  164. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  165. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  166. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  167. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  168. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  169. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  170. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  171. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  172. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  173. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  174. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  175. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  176. metaflow-stubs/plugins/perimeters.pyi +1 -1
  177. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  178. metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
  179. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  180. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  181. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  182. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  183. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  184. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  185. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  186. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  187. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  188. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  189. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  190. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  191. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  192. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
  193. metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
  194. metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
  195. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  196. metaflow-stubs/plugins/uv/uv_environment.pyi +1 -1
  197. metaflow-stubs/profilers/__init__.pyi +1 -1
  198. metaflow-stubs/pylint_wrapper.pyi +1 -1
  199. metaflow-stubs/runner/__init__.pyi +1 -1
  200. metaflow-stubs/runner/deployer.pyi +5 -5
  201. metaflow-stubs/runner/deployer_impl.pyi +1 -1
  202. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  203. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  204. metaflow-stubs/runner/nbrun.pyi +1 -1
  205. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  206. metaflow-stubs/runner/utils.pyi +3 -3
  207. metaflow-stubs/system/__init__.pyi +1 -1
  208. metaflow-stubs/system/system_logger.pyi +1 -1
  209. metaflow-stubs/system/system_monitor.pyi +1 -1
  210. metaflow-stubs/tagging_util.pyi +1 -1
  211. metaflow-stubs/tuple_util.pyi +1 -1
  212. metaflow-stubs/user_configs/__init__.pyi +1 -1
  213. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  214. metaflow-stubs/user_configs/config_options.pyi +2 -2
  215. metaflow-stubs/user_configs/config_parameters.pyi +7 -7
  216. {ob_metaflow_stubs-6.0.3.180rc2.dist-info → ob_metaflow_stubs-6.0.3.180rc4.dist-info}/METADATA +1 -1
  217. ob_metaflow_stubs-6.0.3.180rc4.dist-info/RECORD +220 -0
  218. ob_metaflow_stubs-6.0.3.180rc2.dist-info/RECORD +0 -220
  219. {ob_metaflow_stubs-6.0.3.180rc2.dist-info → ob_metaflow_stubs-6.0.3.180rc4.dist-info}/WHEEL +0 -0
  220. {ob_metaflow_stubs-6.0.3.180rc2.dist-info → ob_metaflow_stubs-6.0.3.180rc4.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.15.17.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-06-17T09:53:19.248370 #
4
+ # Generated on 2025-06-17T18:51:01.431227 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -35,17 +35,17 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
- from . import tuple_util as tuple_util
39
38
  from . import cards as cards
40
- from . import events as events
41
39
  from . import metaflow_git as metaflow_git
40
+ from . import tuple_util as tuple_util
41
+ from . import events as events
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
48
47
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
48
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
49
49
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
@@ -155,70 +155,268 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
155
155
  """
156
156
  ...
157
157
 
158
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
159
+ """
160
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
161
+
162
+ User code call
163
+ --------------
164
+ @ollama(
165
+ models=[...],
166
+ ...
167
+ )
168
+
169
+ Valid backend options
170
+ ---------------------
171
+ - 'local': Run as a separate process on the local task machine.
172
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
173
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
174
+
175
+ Valid model options
176
+ -------------------
177
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
178
+
179
+
180
+ Parameters
181
+ ----------
182
+ models: list[str]
183
+ List of Ollama containers running models in sidecars.
184
+ backend: str
185
+ Determines where and how to run the Ollama process.
186
+ force_pull: bool
187
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
188
+ cache_update_policy: str
189
+ Cache update policy: "auto", "force", or "never".
190
+ force_cache_update: bool
191
+ Simple override for "force" cache update policy.
192
+ debug: bool
193
+ Whether to turn on verbose debugging logs.
194
+ circuit_breaker_config: dict
195
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
196
+ timeout_config: dict
197
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
198
+ """
199
+ ...
200
+
158
201
  @typing.overload
159
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
202
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
160
203
  """
161
- Specifies environment variables to be set prior to the execution of a step.
204
+ Specifies the PyPI packages for the step.
205
+
206
+ Information in this decorator will augment any
207
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
208
+ you can use `@pypi_base` to set packages required by all
209
+ steps and use `@pypi` to specify step-specific overrides.
162
210
 
163
211
 
164
212
  Parameters
165
213
  ----------
166
- vars : Dict[str, str], default {}
167
- Dictionary of environment variables to set.
214
+ packages : Dict[str, str], default: {}
215
+ Packages to use for this step. The key is the name of the package
216
+ and the value is the version to use.
217
+ python : str, optional, default: None
218
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
219
+ that the version used will correspond to the version of the Python interpreter used to start the run.
168
220
  """
169
221
  ...
170
222
 
171
223
  @typing.overload
172
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
224
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
173
225
  ...
174
226
 
175
227
  @typing.overload
176
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
228
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
177
229
  ...
178
230
 
179
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
231
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
180
232
  """
181
- Specifies environment variables to be set prior to the execution of a step.
233
+ Specifies the PyPI packages for the step.
234
+
235
+ Information in this decorator will augment any
236
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
237
+ you can use `@pypi_base` to set packages required by all
238
+ steps and use `@pypi` to specify step-specific overrides.
182
239
 
183
240
 
184
241
  Parameters
185
242
  ----------
186
- vars : Dict[str, str], default {}
187
- Dictionary of environment variables to set.
243
+ packages : Dict[str, str], default: {}
244
+ Packages to use for this step. The key is the name of the package
245
+ and the value is the version to use.
246
+ python : str, optional, default: None
247
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
248
+ that the version used will correspond to the version of the Python interpreter used to start the run.
249
+ """
250
+ ...
251
+
252
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
253
+ """
254
+ Specifies that this step should execute on DGX cloud.
255
+
256
+
257
+ Parameters
258
+ ----------
259
+ gpu : int
260
+ Number of GPUs to use.
261
+ gpu_type : str
262
+ Type of Nvidia GPU to use.
263
+ queue_timeout : int
264
+ Time to keep the job in NVCF's queue.
265
+ """
266
+ ...
267
+
268
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
269
+ """
270
+ Specifies that this step should execute on DGX cloud.
271
+
272
+
273
+ Parameters
274
+ ----------
275
+ gpu : int
276
+ Number of GPUs to use.
277
+ gpu_type : str
278
+ Type of Nvidia GPU to use.
188
279
  """
189
280
  ...
190
281
 
191
282
  @typing.overload
192
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
283
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
193
284
  """
194
- Decorator prototype for all step decorators. This function gets specialized
195
- and imported for all decorators types by _import_plugin_decorators().
285
+ Specifies the number of times the task corresponding
286
+ to a step needs to be retried.
287
+
288
+ This decorator is useful for handling transient errors, such as networking issues.
289
+ If your task contains operations that can't be retried safely, e.g. database updates,
290
+ it is advisable to annotate it with `@retry(times=0)`.
291
+
292
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
293
+ decorator will execute a no-op task after all retries have been exhausted,
294
+ ensuring that the flow execution can continue.
295
+
296
+
297
+ Parameters
298
+ ----------
299
+ times : int, default 3
300
+ Number of times to retry this task.
301
+ minutes_between_retries : int, default 2
302
+ Number of minutes between retries.
196
303
  """
197
304
  ...
198
305
 
199
306
  @typing.overload
200
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
307
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
201
308
  ...
202
309
 
203
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
310
+ @typing.overload
311
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
312
+ ...
313
+
314
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
204
315
  """
205
- Decorator prototype for all step decorators. This function gets specialized
206
- and imported for all decorators types by _import_plugin_decorators().
316
+ Specifies the number of times the task corresponding
317
+ to a step needs to be retried.
318
+
319
+ This decorator is useful for handling transient errors, such as networking issues.
320
+ If your task contains operations that can't be retried safely, e.g. database updates,
321
+ it is advisable to annotate it with `@retry(times=0)`.
322
+
323
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
324
+ decorator will execute a no-op task after all retries have been exhausted,
325
+ ensuring that the flow execution can continue.
326
+
327
+
328
+ Parameters
329
+ ----------
330
+ times : int, default 3
331
+ Number of times to retry this task.
332
+ minutes_between_retries : int, default 2
333
+ Number of minutes between retries.
207
334
  """
208
335
  ...
209
336
 
210
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
337
+ @typing.overload
338
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
211
339
  """
212
- Specifies that this step is used to deploy an instance of the app.
213
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
340
+ Specifies that the step will success under all circumstances.
341
+
342
+ The decorator will create an optional artifact, specified by `var`, which
343
+ contains the exception raised. You can use it to detect the presence
344
+ of errors, indicating that all happy-path artifacts produced by the step
345
+ are missing.
214
346
 
215
347
 
216
348
  Parameters
217
349
  ----------
218
- app_port : int
219
- Number of GPUs to use.
220
- app_name : str
221
- Name of the app to deploy.
350
+ var : str, optional, default None
351
+ Name of the artifact in which to store the caught exception.
352
+ If not specified, the exception is not stored.
353
+ print_exception : bool, default True
354
+ Determines whether or not the exception is printed to
355
+ stdout when caught.
356
+ """
357
+ ...
358
+
359
+ @typing.overload
360
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
361
+ ...
362
+
363
+ @typing.overload
364
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
365
+ ...
366
+
367
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
368
+ """
369
+ Specifies that the step will success under all circumstances.
370
+
371
+ The decorator will create an optional artifact, specified by `var`, which
372
+ contains the exception raised. You can use it to detect the presence
373
+ of errors, indicating that all happy-path artifacts produced by the step
374
+ are missing.
375
+
376
+
377
+ Parameters
378
+ ----------
379
+ var : str, optional, default None
380
+ Name of the artifact in which to store the caught exception.
381
+ If not specified, the exception is not stored.
382
+ print_exception : bool, default True
383
+ Determines whether or not the exception is printed to
384
+ stdout when caught.
385
+ """
386
+ ...
387
+
388
+ @typing.overload
389
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
390
+ """
391
+ Specifies secrets to be retrieved and injected as environment variables prior to
392
+ the execution of a step.
393
+
394
+
395
+ Parameters
396
+ ----------
397
+ sources : List[Union[str, Dict[str, Any]]], default: []
398
+ List of secret specs, defining how the secrets are to be retrieved
399
+ """
400
+ ...
401
+
402
+ @typing.overload
403
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
404
+ ...
405
+
406
+ @typing.overload
407
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
408
+ ...
409
+
410
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
411
+ """
412
+ Specifies secrets to be retrieved and injected as environment variables prior to
413
+ the execution of a step.
414
+
415
+
416
+ Parameters
417
+ ----------
418
+ sources : List[Union[str, Dict[str, Any]]], default: []
419
+ List of secret specs, defining how the secrets are to be retrieved
222
420
  """
223
421
  ...
224
422
 
@@ -279,8 +477,171 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
279
477
  """
280
478
  ...
281
479
 
282
- @typing.overload
283
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
480
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
481
+ """
482
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
483
+
484
+
485
+ Parameters
486
+ ----------
487
+ temp_dir_root : str, optional
488
+ The root directory that will hold the temporary directory where objects will be downloaded.
489
+
490
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
491
+ The list of repos (models/datasets) to load.
492
+
493
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
494
+
495
+ - If repo (model/dataset) is not found in the datastore:
496
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
497
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
498
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
499
+
500
+ - If repo is found in the datastore:
501
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
502
+ """
503
+ ...
504
+
505
+ @typing.overload
506
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
507
+ """
508
+ Specifies the resources needed when executing this step.
509
+
510
+ Use `@resources` to specify the resource requirements
511
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
512
+
513
+ You can choose the compute layer on the command line by executing e.g.
514
+ ```
515
+ python myflow.py run --with batch
516
+ ```
517
+ or
518
+ ```
519
+ python myflow.py run --with kubernetes
520
+ ```
521
+ which executes the flow on the desired system using the
522
+ requirements specified in `@resources`.
523
+
524
+
525
+ Parameters
526
+ ----------
527
+ cpu : int, default 1
528
+ Number of CPUs required for this step.
529
+ gpu : int, optional, default None
530
+ Number of GPUs required for this step.
531
+ disk : int, optional, default None
532
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
533
+ memory : int, default 4096
534
+ Memory size (in MB) required for this step.
535
+ shared_memory : int, optional, default None
536
+ The value for the size (in MiB) of the /dev/shm volume for this step.
537
+ This parameter maps to the `--shm-size` option in Docker.
538
+ """
539
+ ...
540
+
541
+ @typing.overload
542
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
543
+ ...
544
+
545
+ @typing.overload
546
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
547
+ ...
548
+
549
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
550
+ """
551
+ Specifies the resources needed when executing this step.
552
+
553
+ Use `@resources` to specify the resource requirements
554
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
555
+
556
+ You can choose the compute layer on the command line by executing e.g.
557
+ ```
558
+ python myflow.py run --with batch
559
+ ```
560
+ or
561
+ ```
562
+ python myflow.py run --with kubernetes
563
+ ```
564
+ which executes the flow on the desired system using the
565
+ requirements specified in `@resources`.
566
+
567
+
568
+ Parameters
569
+ ----------
570
+ cpu : int, default 1
571
+ Number of CPUs required for this step.
572
+ gpu : int, optional, default None
573
+ Number of GPUs required for this step.
574
+ disk : int, optional, default None
575
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
576
+ memory : int, default 4096
577
+ Memory size (in MB) required for this step.
578
+ shared_memory : int, optional, default None
579
+ The value for the size (in MiB) of the /dev/shm volume for this step.
580
+ This parameter maps to the `--shm-size` option in Docker.
581
+ """
582
+ ...
583
+
584
+ @typing.overload
585
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
586
+ """
587
+ Specifies the Conda environment for the step.
588
+
589
+ Information in this decorator will augment any
590
+ attributes set in the `@conda_base` flow-level decorator. Hence,
591
+ you can use `@conda_base` to set packages required by all
592
+ steps and use `@conda` to specify step-specific overrides.
593
+
594
+
595
+ Parameters
596
+ ----------
597
+ packages : Dict[str, str], default {}
598
+ Packages to use for this step. The key is the name of the package
599
+ and the value is the version to use.
600
+ libraries : Dict[str, str], default {}
601
+ Supported for backward compatibility. When used with packages, packages will take precedence.
602
+ python : str, optional, default None
603
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
604
+ that the version used will correspond to the version of the Python interpreter used to start the run.
605
+ disabled : bool, default False
606
+ If set to True, disables @conda.
607
+ """
608
+ ...
609
+
610
+ @typing.overload
611
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
612
+ ...
613
+
614
+ @typing.overload
615
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
616
+ ...
617
+
618
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
619
+ """
620
+ Specifies the Conda environment for the step.
621
+
622
+ Information in this decorator will augment any
623
+ attributes set in the `@conda_base` flow-level decorator. Hence,
624
+ you can use `@conda_base` to set packages required by all
625
+ steps and use `@conda` to specify step-specific overrides.
626
+
627
+
628
+ Parameters
629
+ ----------
630
+ packages : Dict[str, str], default {}
631
+ Packages to use for this step. The key is the name of the package
632
+ and the value is the version to use.
633
+ libraries : Dict[str, str], default {}
634
+ Supported for backward compatibility. When used with packages, packages will take precedence.
635
+ python : str, optional, default None
636
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
637
+ that the version used will correspond to the version of the Python interpreter used to start the run.
638
+ disabled : bool, default False
639
+ If set to True, disables @conda.
640
+ """
641
+ ...
642
+
643
+ @typing.overload
644
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
284
645
  """
285
646
  Enables checkpointing for a step.
286
647
 
@@ -336,46 +697,22 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
336
697
  """
337
698
  ...
338
699
 
339
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
700
+ @typing.overload
701
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
340
702
  """
341
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
342
-
343
- User code call
344
- --------------
345
- @ollama(
346
- models=[...],
347
- ...
348
- )
349
-
350
- Valid backend options
351
- ---------------------
352
- - 'local': Run as a separate process on the local task machine.
353
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
354
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
355
-
356
- Valid model options
357
- -------------------
358
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
359
-
360
-
361
- Parameters
362
- ----------
363
- models: list[str]
364
- List of Ollama containers running models in sidecars.
365
- backend: str
366
- Determines where and how to run the Ollama process.
367
- force_pull: bool
368
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
369
- cache_update_policy: str
370
- Cache update policy: "auto", "force", or "never".
371
- force_cache_update: bool
372
- Simple override for "force" cache update policy.
373
- debug: bool
374
- Whether to turn on verbose debugging logs.
375
- circuit_breaker_config: dict
376
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
377
- timeout_config: dict
378
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
703
+ Decorator prototype for all step decorators. This function gets specialized
704
+ and imported for all decorators types by _import_plugin_decorators().
705
+ """
706
+ ...
707
+
708
+ @typing.overload
709
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
710
+ ...
711
+
712
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
713
+ """
714
+ Decorator prototype for all step decorators. This function gets specialized
715
+ and imported for all decorators types by _import_plugin_decorators().
379
716
  """
380
717
  ...
381
718
 
@@ -435,263 +772,51 @@ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepF
435
772
  ...
436
773
 
437
774
  @typing.overload
438
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
775
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
439
776
  """
440
- Specifies secrets to be retrieved and injected as environment variables prior to
441
- the execution of a step.
777
+ Creates a human-readable report, a Metaflow Card, after this step completes.
778
+
779
+ Note that you may add multiple `@card` decorators in a step with different parameters.
442
780
 
443
781
 
444
782
  Parameters
445
783
  ----------
446
- sources : List[Union[str, Dict[str, Any]]], default: []
447
- List of secret specs, defining how the secrets are to be retrieved
784
+ type : str, default 'default'
785
+ Card type.
786
+ id : str, optional, default None
787
+ If multiple cards are present, use this id to identify this card.
788
+ options : Dict[str, Any], default {}
789
+ Options passed to the card. The contents depend on the card type.
790
+ timeout : int, default 45
791
+ Interrupt reporting if it takes more than this many seconds.
448
792
  """
449
793
  ...
450
794
 
451
795
  @typing.overload
452
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
796
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
453
797
  ...
454
798
 
455
799
  @typing.overload
456
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
800
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
457
801
  ...
458
802
 
459
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
803
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
460
804
  """
461
- Specifies secrets to be retrieved and injected as environment variables prior to
462
- the execution of a step.
805
+ Creates a human-readable report, a Metaflow Card, after this step completes.
806
+
807
+ Note that you may add multiple `@card` decorators in a step with different parameters.
463
808
 
464
809
 
465
810
  Parameters
466
811
  ----------
467
- sources : List[Union[str, Dict[str, Any]]], default: []
468
- List of secret specs, defining how the secrets are to be retrieved
469
- """
470
- ...
471
-
472
- @typing.overload
473
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
474
- """
475
- Specifies that the step will success under all circumstances.
476
-
477
- The decorator will create an optional artifact, specified by `var`, which
478
- contains the exception raised. You can use it to detect the presence
479
- of errors, indicating that all happy-path artifacts produced by the step
480
- are missing.
481
-
482
-
483
- Parameters
484
- ----------
485
- var : str, optional, default None
486
- Name of the artifact in which to store the caught exception.
487
- If not specified, the exception is not stored.
488
- print_exception : bool, default True
489
- Determines whether or not the exception is printed to
490
- stdout when caught.
491
- """
492
- ...
493
-
494
- @typing.overload
495
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
496
- ...
497
-
498
- @typing.overload
499
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
500
- ...
501
-
502
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
503
- """
504
- Specifies that the step will success under all circumstances.
505
-
506
- The decorator will create an optional artifact, specified by `var`, which
507
- contains the exception raised. You can use it to detect the presence
508
- of errors, indicating that all happy-path artifacts produced by the step
509
- are missing.
510
-
511
-
512
- Parameters
513
- ----------
514
- var : str, optional, default None
515
- Name of the artifact in which to store the caught exception.
516
- If not specified, the exception is not stored.
517
- print_exception : bool, default True
518
- Determines whether or not the exception is printed to
519
- stdout when caught.
520
- """
521
- ...
522
-
523
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
524
- """
525
- Decorator that helps cache, version and store models/datasets from huggingface hub.
526
-
527
-
528
- Parameters
529
- ----------
530
- temp_dir_root : str, optional
531
- The root directory that will hold the temporary directory where objects will be downloaded.
532
-
533
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
534
- The list of repos (models/datasets) to load.
535
-
536
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
537
-
538
- - If repo (model/dataset) is not found in the datastore:
539
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
540
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
541
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
542
-
543
- - If repo is found in the datastore:
544
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
545
- """
546
- ...
547
-
548
- @typing.overload
549
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
550
- """
551
- Specifies the number of times the task corresponding
552
- to a step needs to be retried.
553
-
554
- This decorator is useful for handling transient errors, such as networking issues.
555
- If your task contains operations that can't be retried safely, e.g. database updates,
556
- it is advisable to annotate it with `@retry(times=0)`.
557
-
558
- This can be used in conjunction with the `@catch` decorator. The `@catch`
559
- decorator will execute a no-op task after all retries have been exhausted,
560
- ensuring that the flow execution can continue.
561
-
562
-
563
- Parameters
564
- ----------
565
- times : int, default 3
566
- Number of times to retry this task.
567
- minutes_between_retries : int, default 2
568
- Number of minutes between retries.
569
- """
570
- ...
571
-
572
- @typing.overload
573
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
574
- ...
575
-
576
- @typing.overload
577
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
578
- ...
579
-
580
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
581
- """
582
- Specifies the number of times the task corresponding
583
- to a step needs to be retried.
584
-
585
- This decorator is useful for handling transient errors, such as networking issues.
586
- If your task contains operations that can't be retried safely, e.g. database updates,
587
- it is advisable to annotate it with `@retry(times=0)`.
588
-
589
- This can be used in conjunction with the `@catch` decorator. The `@catch`
590
- decorator will execute a no-op task after all retries have been exhausted,
591
- ensuring that the flow execution can continue.
592
-
593
-
594
- Parameters
595
- ----------
596
- times : int, default 3
597
- Number of times to retry this task.
598
- minutes_between_retries : int, default 2
599
- Number of minutes between retries.
600
- """
601
- ...
602
-
603
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
604
- """
605
- Specifies that this step should execute on DGX cloud.
606
-
607
-
608
- Parameters
609
- ----------
610
- gpu : int
611
- Number of GPUs to use.
612
- gpu_type : str
613
- Type of Nvidia GPU to use.
614
- queue_timeout : int
615
- Time to keep the job in NVCF's queue.
616
- """
617
- ...
618
-
619
- @typing.overload
620
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
621
- """
622
- Specifies the resources needed when executing this step.
623
-
624
- Use `@resources` to specify the resource requirements
625
- independently of the specific compute layer (`@batch`, `@kubernetes`).
626
-
627
- You can choose the compute layer on the command line by executing e.g.
628
- ```
629
- python myflow.py run --with batch
630
- ```
631
- or
632
- ```
633
- python myflow.py run --with kubernetes
634
- ```
635
- which executes the flow on the desired system using the
636
- requirements specified in `@resources`.
637
-
638
-
639
- Parameters
640
- ----------
641
- cpu : int, default 1
642
- Number of CPUs required for this step.
643
- gpu : int, optional, default None
644
- Number of GPUs required for this step.
645
- disk : int, optional, default None
646
- Disk size (in MB) required for this step. Only applies on Kubernetes.
647
- memory : int, default 4096
648
- Memory size (in MB) required for this step.
649
- shared_memory : int, optional, default None
650
- The value for the size (in MiB) of the /dev/shm volume for this step.
651
- This parameter maps to the `--shm-size` option in Docker.
652
- """
653
- ...
654
-
655
- @typing.overload
656
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
657
- ...
658
-
659
- @typing.overload
660
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
661
- ...
662
-
663
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
664
- """
665
- Specifies the resources needed when executing this step.
666
-
667
- Use `@resources` to specify the resource requirements
668
- independently of the specific compute layer (`@batch`, `@kubernetes`).
669
-
670
- You can choose the compute layer on the command line by executing e.g.
671
- ```
672
- python myflow.py run --with batch
673
- ```
674
- or
675
- ```
676
- python myflow.py run --with kubernetes
677
- ```
678
- which executes the flow on the desired system using the
679
- requirements specified in `@resources`.
680
-
681
-
682
- Parameters
683
- ----------
684
- cpu : int, default 1
685
- Number of CPUs required for this step.
686
- gpu : int, optional, default None
687
- Number of GPUs required for this step.
688
- disk : int, optional, default None
689
- Disk size (in MB) required for this step. Only applies on Kubernetes.
690
- memory : int, default 4096
691
- Memory size (in MB) required for this step.
692
- shared_memory : int, optional, default None
693
- The value for the size (in MiB) of the /dev/shm volume for this step.
694
- This parameter maps to the `--shm-size` option in Docker.
812
+ type : str, default 'default'
813
+ Card type.
814
+ id : str, optional, default None
815
+ If multiple cards are present, use this id to identify this card.
816
+ options : Dict[str, Any], default {}
817
+ Options passed to the card. The contents depend on the card type.
818
+ timeout : int, default 45
819
+ Interrupt reporting if it takes more than this many seconds.
695
820
  """
696
821
  ...
697
822
 
@@ -785,175 +910,35 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
785
910
  ...
786
911
 
787
912
  @typing.overload
788
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
789
- """
790
- Specifies the PyPI packages for the step.
791
-
792
- Information in this decorator will augment any
793
- attributes set in the `@pyi_base` flow-level decorator. Hence,
794
- you can use `@pypi_base` to set packages required by all
795
- steps and use `@pypi` to specify step-specific overrides.
796
-
797
-
798
- Parameters
799
- ----------
800
- packages : Dict[str, str], default: {}
801
- Packages to use for this step. The key is the name of the package
802
- and the value is the version to use.
803
- python : str, optional, default: None
804
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
805
- that the version used will correspond to the version of the Python interpreter used to start the run.
806
- """
807
- ...
808
-
809
- @typing.overload
810
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
811
- ...
812
-
813
- @typing.overload
814
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
815
- ...
816
-
817
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
818
- """
819
- Specifies the PyPI packages for the step.
820
-
821
- Information in this decorator will augment any
822
- attributes set in the `@pyi_base` flow-level decorator. Hence,
823
- you can use `@pypi_base` to set packages required by all
824
- steps and use `@pypi` to specify step-specific overrides.
825
-
826
-
827
- Parameters
828
- ----------
829
- packages : Dict[str, str], default: {}
830
- Packages to use for this step. The key is the name of the package
831
- and the value is the version to use.
832
- python : str, optional, default: None
833
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
834
- that the version used will correspond to the version of the Python interpreter used to start the run.
835
- """
836
- ...
837
-
838
- @typing.overload
839
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
840
- """
841
- Creates a human-readable report, a Metaflow Card, after this step completes.
842
-
843
- Note that you may add multiple `@card` decorators in a step with different parameters.
844
-
845
-
846
- Parameters
847
- ----------
848
- type : str, default 'default'
849
- Card type.
850
- id : str, optional, default None
851
- If multiple cards are present, use this id to identify this card.
852
- options : Dict[str, Any], default {}
853
- Options passed to the card. The contents depend on the card type.
854
- timeout : int, default 45
855
- Interrupt reporting if it takes more than this many seconds.
856
- """
857
- ...
858
-
859
- @typing.overload
860
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
861
- ...
862
-
863
- @typing.overload
864
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
865
- ...
866
-
867
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
868
- """
869
- Creates a human-readable report, a Metaflow Card, after this step completes.
870
-
871
- Note that you may add multiple `@card` decorators in a step with different parameters.
872
-
873
-
874
- Parameters
875
- ----------
876
- type : str, default 'default'
877
- Card type.
878
- id : str, optional, default None
879
- If multiple cards are present, use this id to identify this card.
880
- options : Dict[str, Any], default {}
881
- Options passed to the card. The contents depend on the card type.
882
- timeout : int, default 45
883
- Interrupt reporting if it takes more than this many seconds.
884
- """
885
- ...
886
-
887
- @typing.overload
888
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
889
- """
890
- Specifies the Conda environment for the step.
891
-
892
- Information in this decorator will augment any
893
- attributes set in the `@conda_base` flow-level decorator. Hence,
894
- you can use `@conda_base` to set packages required by all
895
- steps and use `@conda` to specify step-specific overrides.
896
-
897
-
898
- Parameters
899
- ----------
900
- packages : Dict[str, str], default {}
901
- Packages to use for this step. The key is the name of the package
902
- and the value is the version to use.
903
- libraries : Dict[str, str], default {}
904
- Supported for backward compatibility. When used with packages, packages will take precedence.
905
- python : str, optional, default None
906
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
907
- that the version used will correspond to the version of the Python interpreter used to start the run.
908
- disabled : bool, default False
909
- If set to True, disables @conda.
910
- """
911
- ...
912
-
913
- @typing.overload
914
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
915
- ...
916
-
917
- @typing.overload
918
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
919
- ...
920
-
921
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
913
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
922
914
  """
923
- Specifies the Conda environment for the step.
924
-
925
- Information in this decorator will augment any
926
- attributes set in the `@conda_base` flow-level decorator. Hence,
927
- you can use `@conda_base` to set packages required by all
928
- steps and use `@conda` to specify step-specific overrides.
915
+ Specifies environment variables to be set prior to the execution of a step.
929
916
 
930
917
 
931
918
  Parameters
932
919
  ----------
933
- packages : Dict[str, str], default {}
934
- Packages to use for this step. The key is the name of the package
935
- and the value is the version to use.
936
- libraries : Dict[str, str], default {}
937
- Supported for backward compatibility. When used with packages, packages will take precedence.
938
- python : str, optional, default None
939
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
940
- that the version used will correspond to the version of the Python interpreter used to start the run.
941
- disabled : bool, default False
942
- If set to True, disables @conda.
920
+ vars : Dict[str, str], default {}
921
+ Dictionary of environment variables to set.
943
922
  """
944
923
  ...
945
924
 
946
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
925
+ @typing.overload
926
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
927
+ ...
928
+
929
+ @typing.overload
930
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
931
+ ...
932
+
933
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
947
934
  """
948
- Specifies that this step should execute on DGX cloud.
935
+ Specifies environment variables to be set prior to the execution of a step.
949
936
 
950
937
 
951
938
  Parameters
952
939
  ----------
953
- gpu : int
954
- Number of GPUs to use.
955
- gpu_type : str
956
- Type of Nvidia GPU to use.
940
+ vars : Dict[str, str], default {}
941
+ Dictionary of environment variables to set.
957
942
  """
958
943
  ...
959
944
 
@@ -1016,6 +1001,21 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
1016
1001
  """
1017
1002
  ...
1018
1003
 
1004
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1005
+ """
1006
+ Specifies that this step is used to deploy an instance of the app.
1007
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
1008
+
1009
+
1010
+ Parameters
1011
+ ----------
1012
+ app_port : int
1013
+ Number of GPUs to use.
1014
+ app_name : str
1015
+ Name of the app to deploy.
1016
+ """
1017
+ ...
1018
+
1019
1019
  @typing.overload
1020
1020
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1021
1021
  """
@@ -1068,43 +1068,53 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1068
1068
  ...
1069
1069
 
1070
1070
  @typing.overload
1071
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1071
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1072
1072
  """
1073
- Specifies the PyPI packages for all steps of the flow.
1073
+ Specifies the times when the flow should be run when running on a
1074
+ production scheduler.
1074
1075
 
1075
- Use `@pypi_base` to set common packages required by all
1076
- steps and use `@pypi` to specify step-specific overrides.
1077
1076
 
1078
1077
  Parameters
1079
1078
  ----------
1080
- packages : Dict[str, str], default: {}
1081
- Packages to use for this flow. The key is the name of the package
1082
- and the value is the version to use.
1083
- python : str, optional, default: None
1084
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1085
- that the version used will correspond to the version of the Python interpreter used to start the run.
1079
+ hourly : bool, default False
1080
+ Run the workflow hourly.
1081
+ daily : bool, default True
1082
+ Run the workflow daily.
1083
+ weekly : bool, default False
1084
+ Run the workflow weekly.
1085
+ cron : str, optional, default None
1086
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1087
+ specified by this expression.
1088
+ timezone : str, optional, default None
1089
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1090
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1086
1091
  """
1087
1092
  ...
1088
1093
 
1089
1094
  @typing.overload
1090
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1095
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1091
1096
  ...
1092
1097
 
1093
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1098
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1094
1099
  """
1095
- Specifies the PyPI packages for all steps of the flow.
1100
+ Specifies the times when the flow should be run when running on a
1101
+ production scheduler.
1096
1102
 
1097
- Use `@pypi_base` to set common packages required by all
1098
- steps and use `@pypi` to specify step-specific overrides.
1099
1103
 
1100
1104
  Parameters
1101
1105
  ----------
1102
- packages : Dict[str, str], default: {}
1103
- Packages to use for this flow. The key is the name of the package
1104
- and the value is the version to use.
1105
- python : str, optional, default: None
1106
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1107
- that the version used will correspond to the version of the Python interpreter used to start the run.
1106
+ hourly : bool, default False
1107
+ Run the workflow hourly.
1108
+ daily : bool, default True
1109
+ Run the workflow daily.
1110
+ weekly : bool, default False
1111
+ Run the workflow weekly.
1112
+ cron : str, optional, default None
1113
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1114
+ specified by this expression.
1115
+ timezone : str, optional, default None
1116
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1117
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1108
1118
  """
1109
1119
  ...
1110
1120
 
@@ -1190,222 +1200,35 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1190
1200
  cp.load(
1191
1201
  latest,
1192
1202
  "test-checkpoints"
1193
- )
1194
-
1195
- task = Task("TorchTuneFlow/8484/train/53673")
1196
- with artifact_store_from(run=run, config={
1197
- "client_params": {
1198
- "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1199
- "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1200
- },
1201
- }):
1202
- load_model(
1203
- task.data.model_ref,
1204
- "test-models"
1205
- )
1206
- ```
1207
- Parameters:
1208
- ----------
1209
-
1210
- type: str
1211
- The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1212
-
1213
- config: dict or Callable
1214
- Dictionary of configuration options for the datastore. The following keys are required:
1215
- - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1216
- - example: 's3://bucket-name/path/to/root'
1217
- - example: 'gs://bucket-name/path/to/root'
1218
- - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1219
- - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1220
- - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1221
- - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1222
- """
1223
- ...
1224
-
1225
- @typing.overload
1226
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1227
- """
1228
- Specifies the event(s) that this flow depends on.
1229
-
1230
- ```
1231
- @trigger(event='foo')
1232
- ```
1233
- or
1234
- ```
1235
- @trigger(events=['foo', 'bar'])
1236
- ```
1237
-
1238
- Additionally, you can specify the parameter mappings
1239
- to map event payload to Metaflow parameters for the flow.
1240
- ```
1241
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1242
- ```
1243
- or
1244
- ```
1245
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1246
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1247
- ```
1248
-
1249
- 'parameters' can also be a list of strings and tuples like so:
1250
- ```
1251
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1252
- ```
1253
- This is equivalent to:
1254
- ```
1255
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1256
- ```
1257
-
1258
-
1259
- Parameters
1260
- ----------
1261
- event : Union[str, Dict[str, Any]], optional, default None
1262
- Event dependency for this flow.
1263
- events : List[Union[str, Dict[str, Any]]], default []
1264
- Events dependency for this flow.
1265
- options : Dict[str, Any], default {}
1266
- Backend-specific configuration for tuning eventing behavior.
1267
- """
1268
- ...
1269
-
1270
- @typing.overload
1271
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1272
- ...
1273
-
1274
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1275
- """
1276
- Specifies the event(s) that this flow depends on.
1277
-
1278
- ```
1279
- @trigger(event='foo')
1280
- ```
1281
- or
1282
- ```
1283
- @trigger(events=['foo', 'bar'])
1284
- ```
1285
-
1286
- Additionally, you can specify the parameter mappings
1287
- to map event payload to Metaflow parameters for the flow.
1288
- ```
1289
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1290
- ```
1291
- or
1292
- ```
1293
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1294
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1295
- ```
1296
-
1297
- 'parameters' can also be a list of strings and tuples like so:
1298
- ```
1299
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1300
- ```
1301
- This is equivalent to:
1302
- ```
1303
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1304
- ```
1305
-
1306
-
1307
- Parameters
1308
- ----------
1309
- event : Union[str, Dict[str, Any]], optional, default None
1310
- Event dependency for this flow.
1311
- events : List[Union[str, Dict[str, Any]]], default []
1312
- Events dependency for this flow.
1313
- options : Dict[str, Any], default {}
1314
- Backend-specific configuration for tuning eventing behavior.
1315
- """
1316
- ...
1317
-
1318
- @typing.overload
1319
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1320
- """
1321
- Specifies the times when the flow should be run when running on a
1322
- production scheduler.
1323
-
1324
-
1325
- Parameters
1326
- ----------
1327
- hourly : bool, default False
1328
- Run the workflow hourly.
1329
- daily : bool, default True
1330
- Run the workflow daily.
1331
- weekly : bool, default False
1332
- Run the workflow weekly.
1333
- cron : str, optional, default None
1334
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1335
- specified by this expression.
1336
- timezone : str, optional, default None
1337
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1338
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1339
- """
1340
- ...
1341
-
1342
- @typing.overload
1343
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1344
- ...
1345
-
1346
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1347
- """
1348
- Specifies the times when the flow should be run when running on a
1349
- production scheduler.
1350
-
1203
+ )
1351
1204
 
1352
- Parameters
1205
+ task = Task("TorchTuneFlow/8484/train/53673")
1206
+ with artifact_store_from(run=run, config={
1207
+ "client_params": {
1208
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
1209
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
1210
+ },
1211
+ }):
1212
+ load_model(
1213
+ task.data.model_ref,
1214
+ "test-models"
1215
+ )
1216
+ ```
1217
+ Parameters:
1353
1218
  ----------
1354
- hourly : bool, default False
1355
- Run the workflow hourly.
1356
- daily : bool, default True
1357
- Run the workflow daily.
1358
- weekly : bool, default False
1359
- Run the workflow weekly.
1360
- cron : str, optional, default None
1361
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1362
- specified by this expression.
1363
- timezone : str, optional, default None
1364
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1365
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1366
- """
1367
- ...
1368
-
1369
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1370
- """
1371
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1372
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1373
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1374
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1375
- starts only after all sensors finish.
1376
1219
 
1220
+ type: str
1221
+ The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
1377
1222
 
1378
- Parameters
1379
- ----------
1380
- timeout : int
1381
- Time, in seconds before the task times out and fails. (Default: 3600)
1382
- poke_interval : int
1383
- Time in seconds that the job should wait in between each try. (Default: 60)
1384
- mode : str
1385
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1386
- exponential_backoff : bool
1387
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1388
- pool : str
1389
- the slot pool this task should run in,
1390
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1391
- soft_fail : bool
1392
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1393
- name : str
1394
- Name of the sensor on Airflow
1395
- description : str
1396
- Description of sensor in the Airflow UI
1397
- bucket_key : Union[str, List[str]]
1398
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1399
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1400
- bucket_name : str
1401
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1402
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1403
- wildcard_match : bool
1404
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1405
- aws_conn_id : str
1406
- a reference to the s3 connection on Airflow. (Default: None)
1407
- verify : bool
1408
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1223
+ config: dict or Callable
1224
+ Dictionary of configuration options for the datastore. The following keys are required:
1225
+ - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
1226
+ - example: 's3://bucket-name/path/to/root'
1227
+ - example: 'gs://bucket-name/path/to/root'
1228
+ - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
1229
+ - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
1230
+ - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
1231
+ - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
1409
1232
  """
1410
1233
  ...
1411
1234
 
@@ -1510,6 +1333,49 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1510
1333
  """
1511
1334
  ...
1512
1335
 
1336
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1337
+ """
1338
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1339
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1340
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1341
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1342
+ starts only after all sensors finish.
1343
+
1344
+
1345
+ Parameters
1346
+ ----------
1347
+ timeout : int
1348
+ Time, in seconds before the task times out and fails. (Default: 3600)
1349
+ poke_interval : int
1350
+ Time in seconds that the job should wait in between each try. (Default: 60)
1351
+ mode : str
1352
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1353
+ exponential_backoff : bool
1354
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1355
+ pool : str
1356
+ the slot pool this task should run in,
1357
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1358
+ soft_fail : bool
1359
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1360
+ name : str
1361
+ Name of the sensor on Airflow
1362
+ description : str
1363
+ Description of sensor in the Airflow UI
1364
+ bucket_key : Union[str, List[str]]
1365
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1366
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1367
+ bucket_name : str
1368
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1369
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1370
+ wildcard_match : bool
1371
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1372
+ aws_conn_id : str
1373
+ a reference to the s3 connection on Airflow. (Default: None)
1374
+ verify : bool
1375
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1376
+ """
1377
+ ...
1378
+
1513
1379
  def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1514
1380
  """
1515
1381
  Specifies what flows belong to the same project.
@@ -1588,5 +1454,139 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1588
1454
  """
1589
1455
  ...
1590
1456
 
1457
+ @typing.overload
1458
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1459
+ """
1460
+ Specifies the PyPI packages for all steps of the flow.
1461
+
1462
+ Use `@pypi_base` to set common packages required by all
1463
+ steps and use `@pypi` to specify step-specific overrides.
1464
+
1465
+ Parameters
1466
+ ----------
1467
+ packages : Dict[str, str], default: {}
1468
+ Packages to use for this flow. The key is the name of the package
1469
+ and the value is the version to use.
1470
+ python : str, optional, default: None
1471
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1472
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1473
+ """
1474
+ ...
1475
+
1476
+ @typing.overload
1477
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1478
+ ...
1479
+
1480
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1481
+ """
1482
+ Specifies the PyPI packages for all steps of the flow.
1483
+
1484
+ Use `@pypi_base` to set common packages required by all
1485
+ steps and use `@pypi` to specify step-specific overrides.
1486
+
1487
+ Parameters
1488
+ ----------
1489
+ packages : Dict[str, str], default: {}
1490
+ Packages to use for this flow. The key is the name of the package
1491
+ and the value is the version to use.
1492
+ python : str, optional, default: None
1493
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1494
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1495
+ """
1496
+ ...
1497
+
1498
+ @typing.overload
1499
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1500
+ """
1501
+ Specifies the event(s) that this flow depends on.
1502
+
1503
+ ```
1504
+ @trigger(event='foo')
1505
+ ```
1506
+ or
1507
+ ```
1508
+ @trigger(events=['foo', 'bar'])
1509
+ ```
1510
+
1511
+ Additionally, you can specify the parameter mappings
1512
+ to map event payload to Metaflow parameters for the flow.
1513
+ ```
1514
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1515
+ ```
1516
+ or
1517
+ ```
1518
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1519
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1520
+ ```
1521
+
1522
+ 'parameters' can also be a list of strings and tuples like so:
1523
+ ```
1524
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1525
+ ```
1526
+ This is equivalent to:
1527
+ ```
1528
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1529
+ ```
1530
+
1531
+
1532
+ Parameters
1533
+ ----------
1534
+ event : Union[str, Dict[str, Any]], optional, default None
1535
+ Event dependency for this flow.
1536
+ events : List[Union[str, Dict[str, Any]]], default []
1537
+ Events dependency for this flow.
1538
+ options : Dict[str, Any], default {}
1539
+ Backend-specific configuration for tuning eventing behavior.
1540
+ """
1541
+ ...
1542
+
1543
+ @typing.overload
1544
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1545
+ ...
1546
+
1547
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1548
+ """
1549
+ Specifies the event(s) that this flow depends on.
1550
+
1551
+ ```
1552
+ @trigger(event='foo')
1553
+ ```
1554
+ or
1555
+ ```
1556
+ @trigger(events=['foo', 'bar'])
1557
+ ```
1558
+
1559
+ Additionally, you can specify the parameter mappings
1560
+ to map event payload to Metaflow parameters for the flow.
1561
+ ```
1562
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1563
+ ```
1564
+ or
1565
+ ```
1566
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1567
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1568
+ ```
1569
+
1570
+ 'parameters' can also be a list of strings and tuples like so:
1571
+ ```
1572
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1573
+ ```
1574
+ This is equivalent to:
1575
+ ```
1576
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1577
+ ```
1578
+
1579
+
1580
+ Parameters
1581
+ ----------
1582
+ event : Union[str, Dict[str, Any]], optional, default None
1583
+ Event dependency for this flow.
1584
+ events : List[Union[str, Dict[str, Any]]], default []
1585
+ Events dependency for this flow.
1586
+ options : Dict[str, Any], default {}
1587
+ Backend-specific configuration for tuning eventing behavior.
1588
+ """
1589
+ ...
1590
+
1591
1591
  pkg_name: str
1592
1592