ob-metaflow-stubs 6.0.3.147__py2.py3-none-any.whl → 6.0.3.150__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. metaflow-stubs/__init__.pyi +599 -600
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +4 -2
  20. metaflow-stubs/metaflow_current.pyi +116 -116
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +4 -4
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +4 -4
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +4 -4
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  61. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  62. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  63. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  74. metaflow-stubs/multicore_utils.pyi +2 -2
  75. metaflow-stubs/parameters.pyi +3 -3
  76. metaflow-stubs/plugins/__init__.pyi +15 -11
  77. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  79. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  80. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  82. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  83. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  84. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  85. metaflow-stubs/plugins/argo/argo_client.pyi +3 -3
  86. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  87. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  88. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  89. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +5 -5
  90. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +7 -5
  91. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  92. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  93. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  94. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  96. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  97. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  100. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  102. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  103. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  104. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  105. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  106. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +4 -4
  107. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  108. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  109. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  110. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  111. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  112. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  113. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  114. metaflow-stubs/plugins/cards/__init__.pyi +6 -6
  115. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  116. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  117. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  118. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  119. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  120. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  121. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  123. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  124. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  125. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  126. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  127. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  128. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  129. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  130. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  131. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  132. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  133. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  134. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  135. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  136. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  137. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  138. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  139. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  140. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  141. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  142. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  143. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  144. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  145. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  146. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  147. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  148. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  150. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  151. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  152. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +4 -2
  153. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  154. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  155. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  156. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  157. metaflow-stubs/plugins/perimeters.pyi +2 -2
  158. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  159. metaflow-stubs/plugins/pypi/__init__.pyi +3 -2
  160. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  161. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  162. metaflow-stubs/plugins/pypi/parsers.pyi +113 -0
  163. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  164. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  165. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  166. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  167. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  168. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  169. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  170. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  171. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  172. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  173. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  174. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  175. metaflow-stubs/profilers/__init__.pyi +2 -2
  176. metaflow-stubs/pylint_wrapper.pyi +2 -2
  177. metaflow-stubs/runner/__init__.pyi +2 -2
  178. metaflow-stubs/runner/deployer.pyi +33 -31
  179. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  180. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  181. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  182. metaflow-stubs/runner/nbrun.pyi +2 -2
  183. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  184. metaflow-stubs/runner/utils.pyi +3 -3
  185. metaflow-stubs/system/__init__.pyi +2 -2
  186. metaflow-stubs/system/system_logger.pyi +3 -3
  187. metaflow-stubs/system/system_monitor.pyi +2 -2
  188. metaflow-stubs/tagging_util.pyi +2 -2
  189. metaflow-stubs/tuple_util.pyi +2 -2
  190. metaflow-stubs/user_configs/__init__.pyi +2 -2
  191. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  192. metaflow-stubs/user_configs/config_options.pyi +3 -3
  193. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  194. {ob_metaflow_stubs-6.0.3.147.dist-info → ob_metaflow_stubs-6.0.3.150.dist-info}/METADATA +1 -1
  195. ob_metaflow_stubs-6.0.3.150.dist-info/RECORD +198 -0
  196. ob_metaflow_stubs-6.0.3.147.dist-info/RECORD +0 -197
  197. {ob_metaflow_stubs-6.0.3.147.dist-info → ob_metaflow_stubs-6.0.3.150.dist-info}/WHEEL +0 -0
  198. {ob_metaflow_stubs-6.0.3.147.dist-info → ob_metaflow_stubs-6.0.3.150.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.0.1+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-03-03T17:55:45.268987 #
3
+ # MF version: 2.15.5.1+obcheckpoint(0.1.9);ob(v1) #
4
+ # Generated on 2025-03-13T17:31:16.313988 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -43,6 +43,9 @@ from . import plugins as plugins
43
43
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
44
44
  from . import includefile as includefile
45
45
  from .includefile import IncludeFile as IncludeFile
46
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
47
+ from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
48
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
46
49
  from . import client as client
47
50
  from .client.core import namespace as namespace
48
51
  from .client.core import get_namespace as get_namespace
@@ -147,61 +150,51 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
147
150
  ...
148
151
 
149
152
  @typing.overload
150
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
153
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
151
154
  """
152
- Specifies a timeout for your step.
153
-
154
- This decorator is useful if this step may hang indefinitely.
155
-
156
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
157
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
158
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
155
+ Creates a human-readable report, a Metaflow Card, after this step completes.
159
156
 
160
- Note that all the values specified in parameters are added together so if you specify
161
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
157
+ Note that you may add multiple `@card` decorators in a step with different parameters.
162
158
 
163
159
 
164
160
  Parameters
165
161
  ----------
166
- seconds : int, default 0
167
- Number of seconds to wait prior to timing out.
168
- minutes : int, default 0
169
- Number of minutes to wait prior to timing out.
170
- hours : int, default 0
171
- Number of hours to wait prior to timing out.
162
+ type : str, default 'default'
163
+ Card type.
164
+ id : str, optional, default None
165
+ If multiple cards are present, use this id to identify this card.
166
+ options : Dict[str, Any], default {}
167
+ Options passed to the card. The contents depend on the card type.
168
+ timeout : int, default 45
169
+ Interrupt reporting if it takes more than this many seconds.
172
170
  """
173
171
  ...
174
172
 
175
173
  @typing.overload
176
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
174
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
177
175
  ...
178
176
 
179
177
  @typing.overload
180
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
178
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
181
179
  ...
182
180
 
183
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
181
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
184
182
  """
185
- Specifies a timeout for your step.
186
-
187
- This decorator is useful if this step may hang indefinitely.
188
-
189
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
190
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
191
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
183
+ Creates a human-readable report, a Metaflow Card, after this step completes.
192
184
 
193
- Note that all the values specified in parameters are added together so if you specify
194
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
185
+ Note that you may add multiple `@card` decorators in a step with different parameters.
195
186
 
196
187
 
197
188
  Parameters
198
189
  ----------
199
- seconds : int, default 0
200
- Number of seconds to wait prior to timing out.
201
- minutes : int, default 0
202
- Number of minutes to wait prior to timing out.
203
- hours : int, default 0
204
- Number of hours to wait prior to timing out.
190
+ type : str, default 'default'
191
+ Card type.
192
+ id : str, optional, default None
193
+ If multiple cards are present, use this id to identify this card.
194
+ options : Dict[str, Any], default {}
195
+ Options passed to the card. The contents depend on the card type.
196
+ timeout : int, default 45
197
+ Interrupt reporting if it takes more than this many seconds.
205
198
  """
206
199
  ...
207
200
 
@@ -265,106 +258,111 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
265
258
  ...
266
259
 
267
260
  @typing.overload
268
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
261
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
269
262
  """
270
- Specifies the resources needed when executing this step.
271
-
272
- Use `@resources` to specify the resource requirements
273
- independently of the specific compute layer (`@batch`, `@kubernetes`).
263
+ Enables checkpointing for a step.
274
264
 
275
- You can choose the compute layer on the command line by executing e.g.
276
- ```
277
- python myflow.py run --with batch
278
- ```
279
- or
280
- ```
281
- python myflow.py run --with kubernetes
282
- ```
283
- which executes the flow on the desired system using the
284
- requirements specified in `@resources`.
285
265
 
286
266
 
287
267
  Parameters
288
268
  ----------
289
- cpu : int, default 1
290
- Number of CPUs required for this step.
291
- gpu : int, optional, default None
292
- Number of GPUs required for this step.
293
- disk : int, optional, default None
294
- Disk size (in MB) required for this step. Only applies on Kubernetes.
295
- memory : int, default 4096
296
- Memory size (in MB) required for this step.
297
- shared_memory : int, optional, default None
298
- The value for the size (in MiB) of the /dev/shm volume for this step.
299
- This parameter maps to the `--shm-size` option in Docker.
269
+ load_policy : str, default: "fresh"
270
+ The policy for loading the checkpoint. The following policies are supported:
271
+ - "eager": Loads the the latest available checkpoint within the namespace.
272
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
273
+ will be loaded at the start of the task.
274
+ - "none": Do not load any checkpoint
275
+ - "fresh": Loads the lastest checkpoint created within the running Task.
276
+ This mode helps loading checkpoints across various retry attempts of the same task.
277
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
278
+ created within the task will be loaded when the task is retries execution on failure.
279
+
280
+ temp_dir_root : str, default: None
281
+ The root directory under which `current.checkpoint.directory` will be created.
300
282
  """
301
283
  ...
302
284
 
303
285
  @typing.overload
304
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
286
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
305
287
  ...
306
288
 
307
289
  @typing.overload
308
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
290
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
309
291
  ...
310
292
 
311
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
293
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
312
294
  """
313
- Specifies the resources needed when executing this step.
314
-
315
- Use `@resources` to specify the resource requirements
316
- independently of the specific compute layer (`@batch`, `@kubernetes`).
295
+ Enables checkpointing for a step.
317
296
 
318
- You can choose the compute layer on the command line by executing e.g.
319
- ```
320
- python myflow.py run --with batch
321
- ```
322
- or
323
- ```
324
- python myflow.py run --with kubernetes
325
- ```
326
- which executes the flow on the desired system using the
327
- requirements specified in `@resources`.
328
297
 
329
298
 
330
299
  Parameters
331
300
  ----------
332
- cpu : int, default 1
333
- Number of CPUs required for this step.
334
- gpu : int, optional, default None
335
- Number of GPUs required for this step.
336
- disk : int, optional, default None
337
- Disk size (in MB) required for this step. Only applies on Kubernetes.
338
- memory : int, default 4096
339
- Memory size (in MB) required for this step.
340
- shared_memory : int, optional, default None
341
- The value for the size (in MiB) of the /dev/shm volume for this step.
342
- This parameter maps to the `--shm-size` option in Docker.
301
+ load_policy : str, default: "fresh"
302
+ The policy for loading the checkpoint. The following policies are supported:
303
+ - "eager": Loads the the latest available checkpoint within the namespace.
304
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
305
+ will be loaded at the start of the task.
306
+ - "none": Do not load any checkpoint
307
+ - "fresh": Loads the lastest checkpoint created within the running Task.
308
+ This mode helps loading checkpoints across various retry attempts of the same task.
309
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
310
+ created within the task will be loaded when the task is retries execution on failure.
311
+
312
+ temp_dir_root : str, default: None
313
+ The root directory under which `current.checkpoint.directory` will be created.
343
314
  """
344
315
  ...
345
316
 
346
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
317
+ @typing.overload
318
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
347
319
  """
348
- Decorator that helps cache, version and store models/datasets from huggingface hub.
349
-
320
+ Decorator prototype for all step decorators. This function gets specialized
321
+ and imported for all decorators types by _import_plugin_decorators().
322
+ """
323
+ ...
324
+
325
+ @typing.overload
326
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
327
+ ...
328
+
329
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
330
+ """
331
+ Decorator prototype for all step decorators. This function gets specialized
332
+ and imported for all decorators types by _import_plugin_decorators().
333
+ """
334
+ ...
335
+
336
+ def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
337
+ """
338
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
350
339
 
351
- Parameters
352
- ----------
353
- temp_dir_root : str, optional
354
- The root directory that will hold the temporary directory where objects will be downloaded.
340
+ User code call
341
+ -----------
342
+ @nim(
343
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
344
+ backend='managed'
345
+ )
355
346
 
356
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
357
- The list of repos (models/datasets) to load.
347
+ Valid backend options
348
+ ---------------------
349
+ - 'managed': Outerbounds selects a compute provider based on the model.
358
350
 
359
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
351
+ Valid model options
352
+ ----------------
353
+ - 'meta/llama3-8b-instruct': 8B parameter model
354
+ - 'meta/llama3-70b-instruct': 70B parameter model
355
+ - any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
360
356
 
361
- - If repo (model/dataset) is not found in the datastore:
362
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
363
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
364
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
365
357
 
366
- - If repo is found in the datastore:
367
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
358
+ Parameters
359
+ ----------
360
+ models: list[NIM]
361
+ List of NIM containers running models in sidecars.
362
+ backend: str
363
+ Compute provider to run the NIM container.
364
+ queue_timeout : int
365
+ Time to keep the job in NVCF's queue.
368
366
  """
369
367
  ...
370
368
 
@@ -474,55 +472,6 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
474
472
  """
475
473
  ...
476
474
 
477
- @typing.overload
478
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
479
- """
480
- Creates a human-readable report, a Metaflow Card, after this step completes.
481
-
482
- Note that you may add multiple `@card` decorators in a step with different parameters.
483
-
484
-
485
- Parameters
486
- ----------
487
- type : str, default 'default'
488
- Card type.
489
- id : str, optional, default None
490
- If multiple cards are present, use this id to identify this card.
491
- options : Dict[str, Any], default {}
492
- Options passed to the card. The contents depend on the card type.
493
- timeout : int, default 45
494
- Interrupt reporting if it takes more than this many seconds.
495
- """
496
- ...
497
-
498
- @typing.overload
499
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
500
- ...
501
-
502
- @typing.overload
503
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
504
- ...
505
-
506
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
507
- """
508
- Creates a human-readable report, a Metaflow Card, after this step completes.
509
-
510
- Note that you may add multiple `@card` decorators in a step with different parameters.
511
-
512
-
513
- Parameters
514
- ----------
515
- type : str, default 'default'
516
- Card type.
517
- id : str, optional, default None
518
- If multiple cards are present, use this id to identify this card.
519
- options : Dict[str, Any], default {}
520
- Options passed to the card. The contents depend on the card type.
521
- timeout : int, default 45
522
- Interrupt reporting if it takes more than this many seconds.
523
- """
524
- ...
525
-
526
475
  @typing.overload
527
476
  def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
528
477
  """
@@ -574,90 +523,28 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
574
523
  """
575
524
  ...
576
525
 
577
- @typing.overload
578
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
526
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
579
527
  """
580
- Internal decorator to support Fast bakery
581
- Parameters
582
- ----------
583
- """
584
- ...
585
-
586
- @typing.overload
587
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
588
- ...
589
-
590
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
591
- """
592
- Internal decorator to support Fast bakery
593
- Parameters
594
- ----------
595
- """
596
- ...
597
-
598
- def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
599
- """
600
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
601
-
602
- User code call
603
- -----------
604
- @ollama(
605
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
606
- backend='local'
607
- )
608
-
609
- Valid backend options
610
- ---------------------
611
- - 'local': Run as a separate process on the local task machine.
612
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
613
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
614
-
615
- Valid model options
616
- ----------------
617
- - 'llama3.2'
618
- - 'llama3.3'
619
- - any model here https://ollama.com/search
528
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
620
529
 
621
530
 
622
531
  Parameters
623
532
  ----------
624
- models: list[Ollama]
625
- List of Ollama containers running models in sidecars.
626
- backend: str
627
- Determines where and how to run the Ollama process.
628
- """
629
- ...
630
-
631
- def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
632
- """
633
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
634
-
635
- User code call
636
- -----------
637
- @nim(
638
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
639
- backend='managed'
640
- )
533
+ temp_dir_root : str, optional
534
+ The root directory that will hold the temporary directory where objects will be downloaded.
641
535
 
642
- Valid backend options
643
- ---------------------
644
- - 'managed': Outerbounds selects a compute provider based on the model.
536
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
537
+ The list of repos (models/datasets) to load.
645
538
 
646
- Valid model options
647
- ----------------
648
- - 'meta/llama3-8b-instruct': 8B parameter model
649
- - 'meta/llama3-70b-instruct': 70B parameter model
650
- - any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
539
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
651
540
 
541
+ - If repo (model/dataset) is not found in the datastore:
542
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
543
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
544
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
652
545
 
653
- Parameters
654
- ----------
655
- models: list[NIM]
656
- List of NIM containers running models in sidecars.
657
- backend: str
658
- Compute provider to run the NIM container.
659
- queue_timeout : int
660
- Time to keep the job in NVCF's queue.
546
+ - If repo is found in the datastore:
547
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
661
548
  """
662
549
  ...
663
550
 
@@ -719,383 +606,435 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
719
606
  ...
720
607
 
721
608
  @typing.overload
722
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
609
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
723
610
  """
724
- Specifies environment variables to be set prior to the execution of a step.
611
+ Specifies secrets to be retrieved and injected as environment variables prior to
612
+ the execution of a step.
725
613
 
726
614
 
727
615
  Parameters
728
616
  ----------
729
- vars : Dict[str, str], default {}
730
- Dictionary of environment variables to set.
617
+ sources : List[Union[str, Dict[str, Any]]], default: []
618
+ List of secret specs, defining how the secrets are to be retrieved
731
619
  """
732
620
  ...
733
621
 
734
622
  @typing.overload
735
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
623
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
736
624
  ...
737
625
 
738
626
  @typing.overload
739
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
627
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
740
628
  ...
741
629
 
742
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
630
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
743
631
  """
744
- Specifies environment variables to be set prior to the execution of a step.
632
+ Specifies secrets to be retrieved and injected as environment variables prior to
633
+ the execution of a step.
745
634
 
746
635
 
747
636
  Parameters
748
637
  ----------
749
- vars : Dict[str, str], default {}
750
- Dictionary of environment variables to set.
638
+ sources : List[Union[str, Dict[str, Any]]], default: []
639
+ List of secret specs, defining how the secrets are to be retrieved
751
640
  """
752
641
  ...
753
642
 
754
- @typing.overload
755
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
643
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
756
644
  """
757
- Enables checkpointing for a step.
758
-
645
+ Specifies that this step should execute on DGX cloud.
759
646
 
760
647
 
761
648
  Parameters
762
649
  ----------
763
- load_policy : str, default: "fresh"
764
- The policy for loading the checkpoint. The following policies are supported:
765
- - "eager": Loads the the latest available checkpoint within the namespace.
766
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
767
- will be loaded at the start of the task.
768
- - "none": Do not load any checkpoint
769
- - "fresh": Loads the lastest checkpoint created within the running Task.
770
- This mode helps loading checkpoints across various retry attempts of the same task.
771
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
772
- created within the task will be loaded when the task is retries execution on failure.
773
-
774
- temp_dir_root : str, default: None
775
- The root directory under which `current.checkpoint.directory` will be created.
650
+ gpu : int
651
+ Number of GPUs to use.
652
+ gpu_type : str
653
+ Type of Nvidia GPU to use.
654
+ queue_timeout : int
655
+ Time to keep the job in NVCF's queue.
776
656
  """
777
657
  ...
778
658
 
779
- @typing.overload
780
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
659
+ def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
660
+ """
661
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
662
+
663
+ User code call
664
+ -----------
665
+ @ollama(
666
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
667
+ backend='local'
668
+ )
669
+
670
+ Valid backend options
671
+ ---------------------
672
+ - 'local': Run as a separate process on the local task machine.
673
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
674
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
675
+
676
+ Valid model options
677
+ ----------------
678
+ - 'llama3.2'
679
+ - 'llama3.3'
680
+ - any model here https://ollama.com/search
681
+
682
+
683
+ Parameters
684
+ ----------
685
+ models: list[Ollama]
686
+ List of Ollama containers running models in sidecars.
687
+ backend: str
688
+ Determines where and how to run the Ollama process.
689
+ """
781
690
  ...
782
691
 
783
692
  @typing.overload
784
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
785
- ...
786
-
787
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
693
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
788
694
  """
789
- Enables checkpointing for a step.
695
+ Specifies the resources needed when executing this step.
696
+
697
+ Use `@resources` to specify the resource requirements
698
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
790
699
 
700
+ You can choose the compute layer on the command line by executing e.g.
701
+ ```
702
+ python myflow.py run --with batch
703
+ ```
704
+ or
705
+ ```
706
+ python myflow.py run --with kubernetes
707
+ ```
708
+ which executes the flow on the desired system using the
709
+ requirements specified in `@resources`.
791
710
 
792
711
 
793
712
  Parameters
794
713
  ----------
795
- load_policy : str, default: "fresh"
796
- The policy for loading the checkpoint. The following policies are supported:
797
- - "eager": Loads the the latest available checkpoint within the namespace.
798
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
799
- will be loaded at the start of the task.
800
- - "none": Do not load any checkpoint
801
- - "fresh": Loads the lastest checkpoint created within the running Task.
802
- This mode helps loading checkpoints across various retry attempts of the same task.
803
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
804
- created within the task will be loaded when the task is retries execution on failure.
805
-
806
- temp_dir_root : str, default: None
807
- The root directory under which `current.checkpoint.directory` will be created.
714
+ cpu : int, default 1
715
+ Number of CPUs required for this step.
716
+ gpu : int, optional, default None
717
+ Number of GPUs required for this step.
718
+ disk : int, optional, default None
719
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
720
+ memory : int, default 4096
721
+ Memory size (in MB) required for this step.
722
+ shared_memory : int, optional, default None
723
+ The value for the size (in MiB) of the /dev/shm volume for this step.
724
+ This parameter maps to the `--shm-size` option in Docker.
808
725
  """
809
726
  ...
810
727
 
811
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
728
+ @typing.overload
729
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
730
+ ...
731
+
732
+ @typing.overload
733
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
734
+ ...
735
+
736
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
812
737
  """
813
- Specifies that this step should execute on Kubernetes.
738
+ Specifies the resources needed when executing this step.
739
+
740
+ Use `@resources` to specify the resource requirements
741
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
742
+
743
+ You can choose the compute layer on the command line by executing e.g.
744
+ ```
745
+ python myflow.py run --with batch
746
+ ```
747
+ or
748
+ ```
749
+ python myflow.py run --with kubernetes
750
+ ```
751
+ which executes the flow on the desired system using the
752
+ requirements specified in `@resources`.
814
753
 
815
754
 
816
755
  Parameters
817
756
  ----------
818
757
  cpu : int, default 1
819
- Number of CPUs required for this step. If `@resources` is
820
- also present, the maximum value from all decorators is used.
758
+ Number of CPUs required for this step.
759
+ gpu : int, optional, default None
760
+ Number of GPUs required for this step.
761
+ disk : int, optional, default None
762
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
821
763
  memory : int, default 4096
822
- Memory size (in MB) required for this step. If
823
- `@resources` is also present, the maximum value from all decorators is
824
- used.
825
- disk : int, default 10240
826
- Disk size (in MB) required for this step. If
827
- `@resources` is also present, the maximum value from all decorators is
828
- used.
829
- image : str, optional, default None
830
- Docker image to use when launching on Kubernetes. If not specified, and
831
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
832
- not, a default Docker image mapping to the current version of Python is used.
833
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
834
- If given, the imagePullPolicy to be applied to the Docker image of the step.
835
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
836
- Kubernetes service account to use when launching pod in Kubernetes.
837
- secrets : List[str], optional, default None
838
- Kubernetes secrets to use when launching pod in Kubernetes. These
839
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
840
- in Metaflow configuration.
841
- node_selector: Union[Dict[str,str], str], optional, default None
842
- Kubernetes node selector(s) to apply to the pod running the task.
843
- Can be passed in as a comma separated string of values e.g.
844
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
845
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
846
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
847
- Kubernetes namespace to use when launching pod in Kubernetes.
848
- gpu : int, optional, default None
849
- Number of GPUs required for this step. A value of zero implies that
850
- the scheduled node should not have GPUs.
851
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
852
- The vendor of the GPUs to be used for this step.
853
- tolerations : List[str], default []
854
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
855
- Kubernetes tolerations to use when launching pod in Kubernetes.
856
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
857
- Kubernetes labels to use when launching pod in Kubernetes.
858
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
859
- Kubernetes annotations to use when launching pod in Kubernetes.
860
- use_tmpfs : bool, default False
861
- This enables an explicit tmpfs mount for this step.
862
- tmpfs_tempdir : bool, default True
863
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
864
- tmpfs_size : int, optional, default: None
865
- The value for the size (in MiB) of the tmpfs mount for this step.
866
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
867
- memory allocated for this step.
868
- tmpfs_path : str, optional, default /metaflow_temp
869
- Path to tmpfs mount for this step.
870
- persistent_volume_claims : Dict[str, str], optional, default None
871
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
872
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
873
- shared_memory: int, optional
874
- Shared memory size (in MiB) required for this step
875
- port: int, optional
876
- Port number to specify in the Kubernetes job object
877
- compute_pool : str, optional, default None
878
- Compute pool to be used for for this step.
879
- If not specified, any accessible compute pool within the perimeter is used.
880
- hostname_resolution_timeout: int, default 10 * 60
881
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
882
- Only applicable when @parallel is used.
883
- qos: str, default: Burstable
884
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
764
+ Memory size (in MB) required for this step.
765
+ shared_memory : int, optional, default None
766
+ The value for the size (in MiB) of the /dev/shm volume for this step.
767
+ This parameter maps to the `--shm-size` option in Docker.
885
768
  """
886
769
  ...
887
770
 
888
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
771
+ @typing.overload
772
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
889
773
  """
890
- Specifies that this step should execute on DGX cloud.
891
-
892
-
893
- Parameters
894
- ----------
895
- gpu : int
896
- Number of GPUs to use.
897
- gpu_type : str
898
- Type of Nvidia GPU to use.
899
- queue_timeout : int
900
- Time to keep the job in NVCF's queue.
774
+ Internal decorator to support Fast bakery
901
775
  """
902
776
  ...
903
777
 
904
778
  @typing.overload
905
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
779
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
780
+ ...
781
+
782
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
906
783
  """
907
- Specifies secrets to be retrieved and injected as environment variables prior to
908
- the execution of a step.
784
+ Internal decorator to support Fast bakery
785
+ """
786
+ ...
787
+
788
+ @typing.overload
789
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
790
+ """
791
+ Specifies environment variables to be set prior to the execution of a step.
909
792
 
910
793
 
911
794
  Parameters
912
795
  ----------
913
- sources : List[Union[str, Dict[str, Any]]], default: []
914
- List of secret specs, defining how the secrets are to be retrieved
796
+ vars : Dict[str, str], default {}
797
+ Dictionary of environment variables to set.
915
798
  """
916
799
  ...
917
800
 
918
801
  @typing.overload
919
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
802
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
920
803
  ...
921
804
 
922
805
  @typing.overload
923
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
806
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
924
807
  ...
925
808
 
926
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
809
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
927
810
  """
928
- Specifies secrets to be retrieved and injected as environment variables prior to
929
- the execution of a step.
811
+ Specifies environment variables to be set prior to the execution of a step.
930
812
 
931
813
 
932
814
  Parameters
933
815
  ----------
934
- sources : List[Union[str, Dict[str, Any]]], default: []
935
- List of secret specs, defining how the secrets are to be retrieved
816
+ vars : Dict[str, str], default {}
817
+ Dictionary of environment variables to set.
936
818
  """
937
819
  ...
938
820
 
939
821
  @typing.overload
940
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
822
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
941
823
  """
824
+ Specifies a timeout for your step.
825
+
826
+ This decorator is useful if this step may hang indefinitely.
827
+
828
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
829
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
830
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
831
+
832
+ Note that all the values specified in parameters are added together so if you specify
833
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
834
+
835
+
942
836
  Parameters
943
837
  ----------
838
+ seconds : int, default 0
839
+ Number of seconds to wait prior to timing out.
840
+ minutes : int, default 0
841
+ Number of minutes to wait prior to timing out.
842
+ hours : int, default 0
843
+ Number of hours to wait prior to timing out.
944
844
  """
945
845
  ...
946
846
 
947
847
  @typing.overload
948
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
848
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
949
849
  ...
950
850
 
951
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
851
+ @typing.overload
852
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
853
+ ...
854
+
855
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
952
856
  """
857
+ Specifies a timeout for your step.
858
+
859
+ This decorator is useful if this step may hang indefinitely.
860
+
861
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
862
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
863
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
864
+
865
+ Note that all the values specified in parameters are added together so if you specify
866
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
867
+
868
+
953
869
  Parameters
954
870
  ----------
871
+ seconds : int, default 0
872
+ Number of seconds to wait prior to timing out.
873
+ minutes : int, default 0
874
+ Number of minutes to wait prior to timing out.
875
+ hours : int, default 0
876
+ Number of hours to wait prior to timing out.
955
877
  """
956
878
  ...
957
879
 
958
- @typing.overload
959
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
880
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
960
881
  """
961
- Specifies the times when the flow should be run when running on a
962
- production scheduler.
882
+ Specifies that this step should execute on Kubernetes.
963
883
 
964
884
 
965
885
  Parameters
966
886
  ----------
967
- hourly : bool, default False
968
- Run the workflow hourly.
969
- daily : bool, default True
970
- Run the workflow daily.
971
- weekly : bool, default False
972
- Run the workflow weekly.
973
- cron : str, optional, default None
974
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
975
- specified by this expression.
976
- timezone : str, optional, default None
977
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
978
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
887
+ cpu : int, default 1
888
+ Number of CPUs required for this step. If `@resources` is
889
+ also present, the maximum value from all decorators is used.
890
+ memory : int, default 4096
891
+ Memory size (in MB) required for this step. If
892
+ `@resources` is also present, the maximum value from all decorators is
893
+ used.
894
+ disk : int, default 10240
895
+ Disk size (in MB) required for this step. If
896
+ `@resources` is also present, the maximum value from all decorators is
897
+ used.
898
+ image : str, optional, default None
899
+ Docker image to use when launching on Kubernetes. If not specified, and
900
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
901
+ not, a default Docker image mapping to the current version of Python is used.
902
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
903
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
904
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
905
+ Kubernetes service account to use when launching pod in Kubernetes.
906
+ secrets : List[str], optional, default None
907
+ Kubernetes secrets to use when launching pod in Kubernetes. These
908
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
909
+ in Metaflow configuration.
910
+ node_selector: Union[Dict[str,str], str], optional, default None
911
+ Kubernetes node selector(s) to apply to the pod running the task.
912
+ Can be passed in as a comma separated string of values e.g.
913
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
914
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
915
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
916
+ Kubernetes namespace to use when launching pod in Kubernetes.
917
+ gpu : int, optional, default None
918
+ Number of GPUs required for this step. A value of zero implies that
919
+ the scheduled node should not have GPUs.
920
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
921
+ The vendor of the GPUs to be used for this step.
922
+ tolerations : List[str], default []
923
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
924
+ Kubernetes tolerations to use when launching pod in Kubernetes.
925
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
926
+ Kubernetes labels to use when launching pod in Kubernetes.
927
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
928
+ Kubernetes annotations to use when launching pod in Kubernetes.
929
+ use_tmpfs : bool, default False
930
+ This enables an explicit tmpfs mount for this step.
931
+ tmpfs_tempdir : bool, default True
932
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
933
+ tmpfs_size : int, optional, default: None
934
+ The value for the size (in MiB) of the tmpfs mount for this step.
935
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
936
+ memory allocated for this step.
937
+ tmpfs_path : str, optional, default /metaflow_temp
938
+ Path to tmpfs mount for this step.
939
+ persistent_volume_claims : Dict[str, str], optional, default None
940
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
941
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
942
+ shared_memory: int, optional
943
+ Shared memory size (in MiB) required for this step
944
+ port: int, optional
945
+ Port number to specify in the Kubernetes job object
946
+ compute_pool : str, optional, default None
947
+ Compute pool to be used for for this step.
948
+ If not specified, any accessible compute pool within the perimeter is used.
949
+ hostname_resolution_timeout: int, default 10 * 60
950
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
951
+ Only applicable when @parallel is used.
952
+ qos: str, default: Burstable
953
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
979
954
  """
980
955
  ...
981
956
 
982
- @typing.overload
983
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
984
- ...
985
-
986
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
957
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
987
958
  """
988
- Specifies the times when the flow should be run when running on a
989
- production scheduler.
959
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
960
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
961
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
962
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
963
+ starts only after all sensors finish.
990
964
 
991
965
 
992
966
  Parameters
993
967
  ----------
994
- hourly : bool, default False
995
- Run the workflow hourly.
996
- daily : bool, default True
997
- Run the workflow daily.
998
- weekly : bool, default False
999
- Run the workflow weekly.
1000
- cron : str, optional, default None
1001
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1002
- specified by this expression.
1003
- timezone : str, optional, default None
1004
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1005
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
968
+ timeout : int
969
+ Time, in seconds before the task times out and fails. (Default: 3600)
970
+ poke_interval : int
971
+ Time in seconds that the job should wait in between each try. (Default: 60)
972
+ mode : str
973
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
974
+ exponential_backoff : bool
975
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
976
+ pool : str
977
+ the slot pool this task should run in,
978
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
979
+ soft_fail : bool
980
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
981
+ name : str
982
+ Name of the sensor on Airflow
983
+ description : str
984
+ Description of sensor in the Airflow UI
985
+ bucket_key : Union[str, List[str]]
986
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
987
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
988
+ bucket_name : str
989
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
990
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
991
+ wildcard_match : bool
992
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
993
+ aws_conn_id : str
994
+ a reference to the s3 connection on Airflow. (Default: None)
995
+ verify : bool
996
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1006
997
  """
1007
998
  ...
1008
999
 
1009
1000
  @typing.overload
1010
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1001
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1011
1002
  """
1012
- Specifies the event(s) that this flow depends on.
1013
-
1014
- ```
1015
- @trigger(event='foo')
1016
- ```
1017
- or
1018
- ```
1019
- @trigger(events=['foo', 'bar'])
1020
- ```
1021
-
1022
- Additionally, you can specify the parameter mappings
1023
- to map event payload to Metaflow parameters for the flow.
1024
- ```
1025
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1026
- ```
1027
- or
1028
- ```
1029
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1030
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1031
- ```
1032
-
1033
- 'parameters' can also be a list of strings and tuples like so:
1034
- ```
1035
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1036
- ```
1037
- This is equivalent to:
1038
- ```
1039
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1040
- ```
1003
+ Specifies the PyPI packages for all steps of the flow.
1041
1004
 
1005
+ Use `@pypi_base` to set common packages required by all
1006
+ steps and use `@pypi` to specify step-specific overrides.
1042
1007
 
1043
1008
  Parameters
1044
1009
  ----------
1045
- event : Union[str, Dict[str, Any]], optional, default None
1046
- Event dependency for this flow.
1047
- events : List[Union[str, Dict[str, Any]]], default []
1048
- Events dependency for this flow.
1049
- options : Dict[str, Any], default {}
1050
- Backend-specific configuration for tuning eventing behavior.
1010
+ packages : Dict[str, str], default: {}
1011
+ Packages to use for this flow. The key is the name of the package
1012
+ and the value is the version to use.
1013
+ python : str, optional, default: None
1014
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1015
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1051
1016
  """
1052
1017
  ...
1053
1018
 
1054
1019
  @typing.overload
1055
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1020
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1056
1021
  ...
1057
1022
 
1058
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1023
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1059
1024
  """
1060
- Specifies the event(s) that this flow depends on.
1061
-
1062
- ```
1063
- @trigger(event='foo')
1064
- ```
1065
- or
1066
- ```
1067
- @trigger(events=['foo', 'bar'])
1068
- ```
1069
-
1070
- Additionally, you can specify the parameter mappings
1071
- to map event payload to Metaflow parameters for the flow.
1072
- ```
1073
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1074
- ```
1075
- or
1076
- ```
1077
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1078
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1079
- ```
1080
-
1081
- 'parameters' can also be a list of strings and tuples like so:
1082
- ```
1083
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1084
- ```
1085
- This is equivalent to:
1086
- ```
1087
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1088
- ```
1025
+ Specifies the PyPI packages for all steps of the flow.
1089
1026
 
1027
+ Use `@pypi_base` to set common packages required by all
1028
+ steps and use `@pypi` to specify step-specific overrides.
1090
1029
 
1091
1030
  Parameters
1092
1031
  ----------
1093
- event : Union[str, Dict[str, Any]], optional, default None
1094
- Event dependency for this flow.
1095
- events : List[Union[str, Dict[str, Any]]], default []
1096
- Events dependency for this flow.
1097
- options : Dict[str, Any], default {}
1098
- Backend-specific configuration for tuning eventing behavior.
1032
+ packages : Dict[str, str], default: {}
1033
+ Packages to use for this flow. The key is the name of the package
1034
+ and the value is the version to use.
1035
+ python : str, optional, default: None
1036
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1037
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1099
1038
  """
1100
1039
  ...
1101
1040
 
@@ -1200,122 +1139,54 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1200
1139
  """
1201
1140
  ...
1202
1141
 
1203
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1204
- """
1205
- Specifies what flows belong to the same project.
1206
-
1207
- A project-specific namespace is created for all flows that
1208
- use the same `@project(name)`.
1209
-
1210
-
1211
- Parameters
1212
- ----------
1213
- name : str
1214
- Project name. Make sure that the name is unique amongst all
1215
- projects that use the same production scheduler. The name may
1216
- contain only lowercase alphanumeric characters and underscores.
1217
-
1218
- branch : Optional[str], default None
1219
- The branch to use. If not specified, the branch is set to
1220
- `user.<username>` unless `production` is set to `True`. This can
1221
- also be set on the command line using `--branch` as a top-level option.
1222
- It is an error to specify `branch` in the decorator and on the command line.
1223
-
1224
- production : bool, default False
1225
- Whether or not the branch is the production branch. This can also be set on the
1226
- command line using `--production` as a top-level option. It is an error to specify
1227
- `production` in the decorator and on the command line.
1228
- The project branch name will be:
1229
- - if `branch` is specified:
1230
- - if `production` is True: `prod.<branch>`
1231
- - if `production` is False: `test.<branch>`
1232
- - if `branch` is not specified:
1233
- - if `production` is True: `prod`
1234
- - if `production` is False: `user.<username>`
1235
- """
1236
- ...
1237
-
1238
1142
  @typing.overload
1239
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1143
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1240
1144
  """
1241
- Specifies the PyPI packages for all steps of the flow.
1145
+ Specifies the Conda environment for all steps of the flow.
1146
+
1147
+ Use `@conda_base` to set common libraries required by all
1148
+ steps and use `@conda` to specify step-specific additions.
1242
1149
 
1243
- Use `@pypi_base` to set common packages required by all
1244
- steps and use `@pypi` to specify step-specific overrides.
1245
1150
 
1246
1151
  Parameters
1247
1152
  ----------
1248
- packages : Dict[str, str], default: {}
1153
+ packages : Dict[str, str], default {}
1249
1154
  Packages to use for this flow. The key is the name of the package
1250
1155
  and the value is the version to use.
1251
- python : str, optional, default: None
1156
+ libraries : Dict[str, str], default {}
1157
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1158
+ python : str, optional, default None
1252
1159
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1253
1160
  that the version used will correspond to the version of the Python interpreter used to start the run.
1161
+ disabled : bool, default False
1162
+ If set to True, disables Conda.
1254
1163
  """
1255
1164
  ...
1256
1165
 
1257
1166
  @typing.overload
1258
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1167
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1259
1168
  ...
1260
1169
 
1261
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1170
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1262
1171
  """
1263
- Specifies the PyPI packages for all steps of the flow.
1172
+ Specifies the Conda environment for all steps of the flow.
1173
+
1174
+ Use `@conda_base` to set common libraries required by all
1175
+ steps and use `@conda` to specify step-specific additions.
1264
1176
 
1265
- Use `@pypi_base` to set common packages required by all
1266
- steps and use `@pypi` to specify step-specific overrides.
1267
1177
 
1268
1178
  Parameters
1269
1179
  ----------
1270
- packages : Dict[str, str], default: {}
1180
+ packages : Dict[str, str], default {}
1271
1181
  Packages to use for this flow. The key is the name of the package
1272
1182
  and the value is the version to use.
1273
- python : str, optional, default: None
1183
+ libraries : Dict[str, str], default {}
1184
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1185
+ python : str, optional, default None
1274
1186
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1275
1187
  that the version used will correspond to the version of the Python interpreter used to start the run.
1276
- """
1277
- ...
1278
-
1279
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1280
- """
1281
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1282
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1283
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1284
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1285
- starts only after all sensors finish.
1286
-
1287
-
1288
- Parameters
1289
- ----------
1290
- timeout : int
1291
- Time, in seconds before the task times out and fails. (Default: 3600)
1292
- poke_interval : int
1293
- Time in seconds that the job should wait in between each try. (Default: 60)
1294
- mode : str
1295
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1296
- exponential_backoff : bool
1297
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1298
- pool : str
1299
- the slot pool this task should run in,
1300
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1301
- soft_fail : bool
1302
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1303
- name : str
1304
- Name of the sensor on Airflow
1305
- description : str
1306
- Description of sensor in the Airflow UI
1307
- bucket_key : Union[str, List[str]]
1308
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1309
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1310
- bucket_name : str
1311
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1312
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1313
- wildcard_match : bool
1314
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1315
- aws_conn_id : str
1316
- a reference to the s3 connection on Airflow. (Default: None)
1317
- verify : bool
1318
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1188
+ disabled : bool, default False
1189
+ If set to True, disables Conda.
1319
1190
  """
1320
1191
  ...
1321
1192
 
@@ -1363,53 +1234,181 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1363
1234
  ...
1364
1235
 
1365
1236
  @typing.overload
1366
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1237
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1367
1238
  """
1368
- Specifies the Conda environment for all steps of the flow.
1239
+ Specifies the event(s) that this flow depends on.
1369
1240
 
1370
- Use `@conda_base` to set common libraries required by all
1371
- steps and use `@conda` to specify step-specific additions.
1241
+ ```
1242
+ @trigger(event='foo')
1243
+ ```
1244
+ or
1245
+ ```
1246
+ @trigger(events=['foo', 'bar'])
1247
+ ```
1248
+
1249
+ Additionally, you can specify the parameter mappings
1250
+ to map event payload to Metaflow parameters for the flow.
1251
+ ```
1252
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1253
+ ```
1254
+ or
1255
+ ```
1256
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1257
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1258
+ ```
1259
+
1260
+ 'parameters' can also be a list of strings and tuples like so:
1261
+ ```
1262
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1263
+ ```
1264
+ This is equivalent to:
1265
+ ```
1266
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1267
+ ```
1372
1268
 
1373
1269
 
1374
1270
  Parameters
1375
1271
  ----------
1376
- packages : Dict[str, str], default {}
1377
- Packages to use for this flow. The key is the name of the package
1378
- and the value is the version to use.
1379
- libraries : Dict[str, str], default {}
1380
- Supported for backward compatibility. When used with packages, packages will take precedence.
1381
- python : str, optional, default None
1382
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1383
- that the version used will correspond to the version of the Python interpreter used to start the run.
1384
- disabled : bool, default False
1385
- If set to True, disables Conda.
1272
+ event : Union[str, Dict[str, Any]], optional, default None
1273
+ Event dependency for this flow.
1274
+ events : List[Union[str, Dict[str, Any]]], default []
1275
+ Events dependency for this flow.
1276
+ options : Dict[str, Any], default {}
1277
+ Backend-specific configuration for tuning eventing behavior.
1386
1278
  """
1387
1279
  ...
1388
1280
 
1389
1281
  @typing.overload
1390
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1282
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1391
1283
  ...
1392
1284
 
1393
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1285
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1394
1286
  """
1395
- Specifies the Conda environment for all steps of the flow.
1287
+ Specifies the event(s) that this flow depends on.
1396
1288
 
1397
- Use `@conda_base` to set common libraries required by all
1398
- steps and use `@conda` to specify step-specific additions.
1289
+ ```
1290
+ @trigger(event='foo')
1291
+ ```
1292
+ or
1293
+ ```
1294
+ @trigger(events=['foo', 'bar'])
1295
+ ```
1296
+
1297
+ Additionally, you can specify the parameter mappings
1298
+ to map event payload to Metaflow parameters for the flow.
1299
+ ```
1300
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1301
+ ```
1302
+ or
1303
+ ```
1304
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1305
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1306
+ ```
1307
+
1308
+ 'parameters' can also be a list of strings and tuples like so:
1309
+ ```
1310
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1311
+ ```
1312
+ This is equivalent to:
1313
+ ```
1314
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1315
+ ```
1399
1316
 
1400
1317
 
1401
1318
  Parameters
1402
1319
  ----------
1403
- packages : Dict[str, str], default {}
1404
- Packages to use for this flow. The key is the name of the package
1405
- and the value is the version to use.
1406
- libraries : Dict[str, str], default {}
1407
- Supported for backward compatibility. When used with packages, packages will take precedence.
1408
- python : str, optional, default None
1409
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1410
- that the version used will correspond to the version of the Python interpreter used to start the run.
1411
- disabled : bool, default False
1412
- If set to True, disables Conda.
1320
+ event : Union[str, Dict[str, Any]], optional, default None
1321
+ Event dependency for this flow.
1322
+ events : List[Union[str, Dict[str, Any]]], default []
1323
+ Events dependency for this flow.
1324
+ options : Dict[str, Any], default {}
1325
+ Backend-specific configuration for tuning eventing behavior.
1326
+ """
1327
+ ...
1328
+
1329
+ @typing.overload
1330
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1331
+ """
1332
+ Specifies the times when the flow should be run when running on a
1333
+ production scheduler.
1334
+
1335
+
1336
+ Parameters
1337
+ ----------
1338
+ hourly : bool, default False
1339
+ Run the workflow hourly.
1340
+ daily : bool, default True
1341
+ Run the workflow daily.
1342
+ weekly : bool, default False
1343
+ Run the workflow weekly.
1344
+ cron : str, optional, default None
1345
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1346
+ specified by this expression.
1347
+ timezone : str, optional, default None
1348
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1349
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1350
+ """
1351
+ ...
1352
+
1353
+ @typing.overload
1354
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1355
+ ...
1356
+
1357
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1358
+ """
1359
+ Specifies the times when the flow should be run when running on a
1360
+ production scheduler.
1361
+
1362
+
1363
+ Parameters
1364
+ ----------
1365
+ hourly : bool, default False
1366
+ Run the workflow hourly.
1367
+ daily : bool, default True
1368
+ Run the workflow daily.
1369
+ weekly : bool, default False
1370
+ Run the workflow weekly.
1371
+ cron : str, optional, default None
1372
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1373
+ specified by this expression.
1374
+ timezone : str, optional, default None
1375
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1376
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1377
+ """
1378
+ ...
1379
+
1380
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1381
+ """
1382
+ Specifies what flows belong to the same project.
1383
+
1384
+ A project-specific namespace is created for all flows that
1385
+ use the same `@project(name)`.
1386
+
1387
+
1388
+ Parameters
1389
+ ----------
1390
+ name : str
1391
+ Project name. Make sure that the name is unique amongst all
1392
+ projects that use the same production scheduler. The name may
1393
+ contain only lowercase alphanumeric characters and underscores.
1394
+
1395
+ branch : Optional[str], default None
1396
+ The branch to use. If not specified, the branch is set to
1397
+ `user.<username>` unless `production` is set to `True`. This can
1398
+ also be set on the command line using `--branch` as a top-level option.
1399
+ It is an error to specify `branch` in the decorator and on the command line.
1400
+
1401
+ production : bool, default False
1402
+ Whether or not the branch is the production branch. This can also be set on the
1403
+ command line using `--production` as a top-level option. It is an error to specify
1404
+ `production` in the decorator and on the command line.
1405
+ The project branch name will be:
1406
+ - if `branch` is specified:
1407
+ - if `production` is True: `prod.<branch>`
1408
+ - if `production` is False: `test.<branch>`
1409
+ - if `branch` is not specified:
1410
+ - if `production` is True: `prod`
1411
+ - if `production` is False: `user.<username>`
1413
1412
  """
1414
1413
  ...
1415
1414