ob-metaflow-stubs 6.0.3.162__py2.py3-none-any.whl → 6.0.3.163__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. metaflow-stubs/__init__.pyi +615 -615
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +93 -93
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +4 -4
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  76. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  78. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  79. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  80. metaflow-stubs/multicore_utils.pyi +2 -2
  81. metaflow-stubs/parameters.pyi +4 -4
  82. metaflow-stubs/plugins/__init__.pyi +11 -11
  83. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  84. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  85. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  86. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  88. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  89. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  90. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  92. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  93. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  94. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  95. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  96. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  97. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  99. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  100. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  102. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  103. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  105. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +5 -5
  106. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  107. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  108. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  109. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  110. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  111. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  112. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  113. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  114. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  115. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  116. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +5 -5
  117. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  118. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  119. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  120. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  121. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  123. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  124. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  126. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  127. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  128. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  129. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  130. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  131. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  132. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  133. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  134. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  135. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  136. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  138. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  139. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  140. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  141. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  142. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  143. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  144. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  145. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  146. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  147. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  148. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  149. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +5 -5
  150. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  151. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  152. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  153. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  154. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  155. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  156. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  157. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  158. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  159. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  160. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  161. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  162. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  163. metaflow-stubs/plugins/perimeters.pyi +2 -2
  164. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  165. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  166. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  167. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  168. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  169. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  170. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  171. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  172. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  173. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  174. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  175. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  176. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  177. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  178. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  179. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  180. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  181. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  182. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  183. metaflow-stubs/profilers/__init__.pyi +2 -2
  184. metaflow-stubs/pylint_wrapper.pyi +2 -2
  185. metaflow-stubs/runner/__init__.pyi +2 -2
  186. metaflow-stubs/runner/deployer.pyi +29 -29
  187. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  188. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  189. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  190. metaflow-stubs/runner/nbrun.pyi +2 -2
  191. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  192. metaflow-stubs/runner/utils.pyi +3 -3
  193. metaflow-stubs/system/__init__.pyi +2 -2
  194. metaflow-stubs/system/system_logger.pyi +3 -3
  195. metaflow-stubs/system/system_monitor.pyi +2 -2
  196. metaflow-stubs/tagging_util.pyi +2 -2
  197. metaflow-stubs/tuple_util.pyi +2 -2
  198. metaflow-stubs/user_configs/__init__.pyi +2 -2
  199. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  200. metaflow-stubs/user_configs/config_options.pyi +3 -3
  201. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  202. {ob_metaflow_stubs-6.0.3.162.dist-info → ob_metaflow_stubs-6.0.3.163.dist-info}/METADATA +1 -1
  203. ob_metaflow_stubs-6.0.3.163.dist-info/RECORD +206 -0
  204. ob_metaflow_stubs-6.0.3.162.dist-info/RECORD +0 -206
  205. {ob_metaflow_stubs-6.0.3.162.dist-info → ob_metaflow_stubs-6.0.3.163.dist-info}/WHEEL +0 -0
  206. {ob_metaflow_stubs-6.0.3.162.dist-info → ob_metaflow_stubs-6.0.3.163.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.10.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-05-05T21:26:32.787199 #
3
+ # MF version: 2.15.11.1+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-05-06T23:22:15.553269 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -37,16 +37,16 @@ from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDec
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
38
  from . import cards as cards
39
39
  from . import tuple_util as tuple_util
40
- from . import metaflow_git as metaflow_git
41
40
  from . import events as events
41
+ from . import metaflow_git as metaflow_git
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
47
48
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
48
49
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
49
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
52
52
  from .client.core import get_namespace as get_namespace
@@ -153,18 +153,87 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
153
153
  """
154
154
  ...
155
155
 
156
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
156
+ @typing.overload
157
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
157
158
  """
158
- Specifies that this step is used to deploy an instance of the app.
159
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
159
+ Specifies that the step will success under all circumstances.
160
+
161
+ The decorator will create an optional artifact, specified by `var`, which
162
+ contains the exception raised. You can use it to detect the presence
163
+ of errors, indicating that all happy-path artifacts produced by the step
164
+ are missing.
160
165
 
161
166
 
162
167
  Parameters
163
168
  ----------
164
- app_port : int
165
- Number of GPUs to use.
166
- app_name : str
167
- Name of the app to deploy.
169
+ var : str, optional, default None
170
+ Name of the artifact in which to store the caught exception.
171
+ If not specified, the exception is not stored.
172
+ print_exception : bool, default True
173
+ Determines whether or not the exception is printed to
174
+ stdout when caught.
175
+ """
176
+ ...
177
+
178
+ @typing.overload
179
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
180
+ ...
181
+
182
+ @typing.overload
183
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
184
+ ...
185
+
186
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
187
+ """
188
+ Specifies that the step will success under all circumstances.
189
+
190
+ The decorator will create an optional artifact, specified by `var`, which
191
+ contains the exception raised. You can use it to detect the presence
192
+ of errors, indicating that all happy-path artifacts produced by the step
193
+ are missing.
194
+
195
+
196
+ Parameters
197
+ ----------
198
+ var : str, optional, default None
199
+ Name of the artifact in which to store the caught exception.
200
+ If not specified, the exception is not stored.
201
+ print_exception : bool, default True
202
+ Determines whether or not the exception is printed to
203
+ stdout when caught.
204
+ """
205
+ ...
206
+
207
+ @typing.overload
208
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
209
+ """
210
+ Specifies environment variables to be set prior to the execution of a step.
211
+
212
+
213
+ Parameters
214
+ ----------
215
+ vars : Dict[str, str], default {}
216
+ Dictionary of environment variables to set.
217
+ """
218
+ ...
219
+
220
+ @typing.overload
221
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
222
+ ...
223
+
224
+ @typing.overload
225
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
226
+ ...
227
+
228
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
229
+ """
230
+ Specifies environment variables to be set prior to the execution of a step.
231
+
232
+
233
+ Parameters
234
+ ----------
235
+ vars : Dict[str, str], default {}
236
+ Dictionary of environment variables to set.
168
237
  """
169
238
  ...
170
239
 
@@ -194,68 +263,105 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
194
263
  ...
195
264
 
196
265
  @typing.overload
197
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
266
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
198
267
  """
199
- Internal decorator to support Fast bakery
268
+ Enables loading / saving of models within a step.
269
+
270
+
271
+
272
+ Parameters
273
+ ----------
274
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
275
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
276
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
277
+ - `current.checkpoint`
278
+ - `current.model`
279
+ - `current.huggingface_hub`
280
+
281
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
282
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
283
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
284
+
285
+ temp_dir_root : str, default: None
286
+ The root directory under which `current.model.loaded` will store loaded models
200
287
  """
201
288
  ...
202
289
 
203
290
  @typing.overload
204
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
291
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
205
292
  ...
206
293
 
207
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
208
- """
209
- Internal decorator to support Fast bakery
210
- """
294
+ @typing.overload
295
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
211
296
  ...
212
297
 
213
- @typing.overload
214
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
298
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
215
299
  """
216
- Specifies secrets to be retrieved and injected as environment variables prior to
217
- the execution of a step.
300
+ Enables loading / saving of models within a step.
301
+
218
302
 
219
303
 
220
304
  Parameters
221
305
  ----------
222
- sources : List[Union[str, Dict[str, Any]]], default: []
223
- List of secret specs, defining how the secrets are to be retrieved
306
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
307
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
308
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
309
+ - `current.checkpoint`
310
+ - `current.model`
311
+ - `current.huggingface_hub`
312
+
313
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
314
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
315
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
316
+
317
+ temp_dir_root : str, default: None
318
+ The root directory under which `current.model.loaded` will store loaded models
224
319
  """
225
320
  ...
226
321
 
227
- @typing.overload
228
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
229
- ...
230
-
231
- @typing.overload
232
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
233
- ...
234
-
235
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
322
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
236
323
  """
237
- Specifies secrets to be retrieved and injected as environment variables prior to
238
- the execution of a step.
324
+ Specifies that this step is used to deploy an instance of the app.
325
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
239
326
 
240
327
 
241
328
  Parameters
242
329
  ----------
243
- sources : List[Union[str, Dict[str, Any]]], default: []
244
- List of secret specs, defining how the secrets are to be retrieved
330
+ app_port : int
331
+ Number of GPUs to use.
332
+ app_name : str
333
+ Name of the app to deploy.
245
334
  """
246
335
  ...
247
336
 
248
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
337
+ def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
249
338
  """
250
- Specifies that this step should execute on DGX cloud.
339
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
340
+
341
+ User code call
342
+ -----------
343
+ @nim(
344
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
345
+ backend='managed'
346
+ )
347
+
348
+ Valid backend options
349
+ ---------------------
350
+ - 'managed': Outerbounds selects a compute provider based on the model.
351
+
352
+ Valid model options
353
+ ----------------
354
+ - 'meta/llama3-8b-instruct': 8B parameter model
355
+ - 'meta/llama3-70b-instruct': 70B parameter model
356
+ - any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
251
357
 
252
358
 
253
359
  Parameters
254
360
  ----------
255
- gpu : int
256
- Number of GPUs to use.
257
- gpu_type : str
258
- Type of Nvidia GPU to use.
361
+ models: list[NIM]
362
+ List of NIM containers running models in sidecars.
363
+ backend: str
364
+ Compute provider to run the NIM container.
259
365
  queue_timeout : int
260
366
  Time to keep the job in NVCF's queue.
261
367
  """
@@ -380,91 +486,154 @@ def ollama(*, models: "list[Ollama]", backend: str) -> typing.Callable[[typing.U
380
486
  ...
381
487
 
382
488
  @typing.overload
383
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
489
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
384
490
  """
385
- Decorator prototype for all step decorators. This function gets specialized
386
- and imported for all decorators types by _import_plugin_decorators().
491
+ Internal decorator to support Fast bakery
387
492
  """
388
493
  ...
389
494
 
390
495
  @typing.overload
391
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
496
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
392
497
  ...
393
498
 
394
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
499
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
395
500
  """
396
- Decorator prototype for all step decorators. This function gets specialized
397
- and imported for all decorators types by _import_plugin_decorators().
501
+ Internal decorator to support Fast bakery
398
502
  """
399
503
  ...
400
504
 
401
- def nim(*, models: "list[NIM]", backend: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
505
+ @typing.overload
506
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
402
507
  """
403
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
404
-
405
- User code call
406
- -----------
407
- @nim(
408
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
409
- backend='managed'
410
- )
411
-
412
- Valid backend options
413
- ---------------------
414
- - 'managed': Outerbounds selects a compute provider based on the model.
508
+ Specifies the PyPI packages for the step.
415
509
 
416
- Valid model options
417
- ----------------
418
- - 'meta/llama3-8b-instruct': 8B parameter model
419
- - 'meta/llama3-70b-instruct': 70B parameter model
420
- - any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
510
+ Information in this decorator will augment any
511
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
512
+ you can use `@pypi_base` to set packages required by all
513
+ steps and use `@pypi` to specify step-specific overrides.
421
514
 
422
515
 
423
516
  Parameters
424
517
  ----------
425
- models: list[NIM]
426
- List of NIM containers running models in sidecars.
427
- backend: str
428
- Compute provider to run the NIM container.
429
- queue_timeout : int
430
- Time to keep the job in NVCF's queue.
518
+ packages : Dict[str, str], default: {}
519
+ Packages to use for this step. The key is the name of the package
520
+ and the value is the version to use.
521
+ python : str, optional, default: None
522
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
523
+ that the version used will correspond to the version of the Python interpreter used to start the run.
431
524
  """
432
525
  ...
433
526
 
434
527
  @typing.overload
435
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
528
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
529
+ ...
530
+
531
+ @typing.overload
532
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
533
+ ...
534
+
535
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
436
536
  """
437
- Enables checkpointing for a step.
537
+ Specifies the PyPI packages for the step.
438
538
 
539
+ Information in this decorator will augment any
540
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
541
+ you can use `@pypi_base` to set packages required by all
542
+ steps and use `@pypi` to specify step-specific overrides.
439
543
 
440
544
 
441
545
  Parameters
442
546
  ----------
443
- load_policy : str, default: "fresh"
444
- The policy for loading the checkpoint. The following policies are supported:
445
- - "eager": Loads the the latest available checkpoint within the namespace.
446
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
447
- will be loaded at the start of the task.
448
- - "none": Do not load any checkpoint
449
- - "fresh": Loads the lastest checkpoint created within the running Task.
450
- This mode helps loading checkpoints across various retry attempts of the same task.
451
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
452
- created within the task will be loaded when the task is retries execution on failure.
547
+ packages : Dict[str, str], default: {}
548
+ Packages to use for this step. The key is the name of the package
549
+ and the value is the version to use.
550
+ python : str, optional, default: None
551
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
552
+ that the version used will correspond to the version of the Python interpreter used to start the run.
553
+ """
554
+ ...
555
+
556
+ @typing.overload
557
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
558
+ """
559
+ Specifies the resources needed when executing this step.
453
560
 
454
- temp_dir_root : str, default: None
455
- The root directory under which `current.checkpoint.directory` will be created.
561
+ Use `@resources` to specify the resource requirements
562
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
563
+
564
+ You can choose the compute layer on the command line by executing e.g.
565
+ ```
566
+ python myflow.py run --with batch
567
+ ```
568
+ or
569
+ ```
570
+ python myflow.py run --with kubernetes
571
+ ```
572
+ which executes the flow on the desired system using the
573
+ requirements specified in `@resources`.
574
+
575
+
576
+ Parameters
577
+ ----------
578
+ cpu : int, default 1
579
+ Number of CPUs required for this step.
580
+ gpu : int, optional, default None
581
+ Number of GPUs required for this step.
582
+ disk : int, optional, default None
583
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
584
+ memory : int, default 4096
585
+ Memory size (in MB) required for this step.
586
+ shared_memory : int, optional, default None
587
+ The value for the size (in MiB) of the /dev/shm volume for this step.
588
+ This parameter maps to the `--shm-size` option in Docker.
456
589
  """
457
590
  ...
458
591
 
459
592
  @typing.overload
460
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
593
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
461
594
  ...
462
595
 
463
596
  @typing.overload
464
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
597
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
465
598
  ...
466
599
 
467
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
600
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
601
+ """
602
+ Specifies the resources needed when executing this step.
603
+
604
+ Use `@resources` to specify the resource requirements
605
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
606
+
607
+ You can choose the compute layer on the command line by executing e.g.
608
+ ```
609
+ python myflow.py run --with batch
610
+ ```
611
+ or
612
+ ```
613
+ python myflow.py run --with kubernetes
614
+ ```
615
+ which executes the flow on the desired system using the
616
+ requirements specified in `@resources`.
617
+
618
+
619
+ Parameters
620
+ ----------
621
+ cpu : int, default 1
622
+ Number of CPUs required for this step.
623
+ gpu : int, optional, default None
624
+ Number of GPUs required for this step.
625
+ disk : int, optional, default None
626
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
627
+ memory : int, default 4096
628
+ Memory size (in MB) required for this step.
629
+ shared_memory : int, optional, default None
630
+ The value for the size (in MiB) of the /dev/shm volume for this step.
631
+ This parameter maps to the `--shm-size` option in Docker.
632
+ """
633
+ ...
634
+
635
+ @typing.overload
636
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
468
637
  """
469
638
  Enables checkpointing for a step.
470
639
 
@@ -489,108 +658,53 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
489
658
  ...
490
659
 
491
660
  @typing.overload
492
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
493
- """
494
- Specifies that the step will success under all circumstances.
495
-
496
- The decorator will create an optional artifact, specified by `var`, which
497
- contains the exception raised. You can use it to detect the presence
498
- of errors, indicating that all happy-path artifacts produced by the step
499
- are missing.
500
-
501
-
502
- Parameters
503
- ----------
504
- var : str, optional, default None
505
- Name of the artifact in which to store the caught exception.
506
- If not specified, the exception is not stored.
507
- print_exception : bool, default True
508
- Determines whether or not the exception is printed to
509
- stdout when caught.
510
- """
511
- ...
512
-
513
- @typing.overload
514
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
661
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
515
662
  ...
516
663
 
517
664
  @typing.overload
518
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
665
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
519
666
  ...
520
667
 
521
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
668
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
522
669
  """
523
- Specifies that the step will success under all circumstances.
670
+ Enables checkpointing for a step.
524
671
 
525
- The decorator will create an optional artifact, specified by `var`, which
526
- contains the exception raised. You can use it to detect the presence
527
- of errors, indicating that all happy-path artifacts produced by the step
528
- are missing.
529
672
 
530
673
 
531
674
  Parameters
532
675
  ----------
533
- var : str, optional, default None
534
- Name of the artifact in which to store the caught exception.
535
- If not specified, the exception is not stored.
536
- print_exception : bool, default True
537
- Determines whether or not the exception is printed to
538
- stdout when caught.
676
+ load_policy : str, default: "fresh"
677
+ The policy for loading the checkpoint. The following policies are supported:
678
+ - "eager": Loads the the latest available checkpoint within the namespace.
679
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
680
+ will be loaded at the start of the task.
681
+ - "none": Do not load any checkpoint
682
+ - "fresh": Loads the lastest checkpoint created within the running Task.
683
+ This mode helps loading checkpoints across various retry attempts of the same task.
684
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
685
+ created within the task will be loaded when the task is retries execution on failure.
686
+
687
+ temp_dir_root : str, default: None
688
+ The root directory under which `current.checkpoint.directory` will be created.
539
689
  """
540
690
  ...
541
691
 
542
692
  @typing.overload
543
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
693
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
544
694
  """
545
- Specifies the number of times the task corresponding
546
- to a step needs to be retried.
547
-
548
- This decorator is useful for handling transient errors, such as networking issues.
549
- If your task contains operations that can't be retried safely, e.g. database updates,
550
- it is advisable to annotate it with `@retry(times=0)`.
551
-
552
- This can be used in conjunction with the `@catch` decorator. The `@catch`
553
- decorator will execute a no-op task after all retries have been exhausted,
554
- ensuring that the flow execution can continue.
555
-
556
-
557
- Parameters
558
- ----------
559
- times : int, default 3
560
- Number of times to retry this task.
561
- minutes_between_retries : int, default 2
562
- Number of minutes between retries.
695
+ Decorator prototype for all step decorators. This function gets specialized
696
+ and imported for all decorators types by _import_plugin_decorators().
563
697
  """
564
698
  ...
565
699
 
566
700
  @typing.overload
567
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
568
- ...
569
-
570
- @typing.overload
571
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
701
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
572
702
  ...
573
703
 
574
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
704
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
575
705
  """
576
- Specifies the number of times the task corresponding
577
- to a step needs to be retried.
578
-
579
- This decorator is useful for handling transient errors, such as networking issues.
580
- If your task contains operations that can't be retried safely, e.g. database updates,
581
- it is advisable to annotate it with `@retry(times=0)`.
582
-
583
- This can be used in conjunction with the `@catch` decorator. The `@catch`
584
- decorator will execute a no-op task after all retries have been exhausted,
585
- ensuring that the flow execution can continue.
586
-
587
-
588
- Parameters
589
- ----------
590
- times : int, default 3
591
- Number of times to retry this task.
592
- minutes_between_retries : int, default 2
593
- Number of minutes between retries.
706
+ Decorator prototype for all step decorators. This function gets specialized
707
+ and imported for all decorators types by _import_plugin_decorators().
594
708
  """
595
709
  ...
596
710
 
@@ -702,317 +816,168 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
702
816
  """
703
817
  ...
704
818
 
705
- @typing.overload
706
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
819
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
707
820
  """
708
- Specifies a timeout for your step.
821
+ Specifies that this step should execute on DGX cloud.
709
822
 
710
- This decorator is useful if this step may hang indefinitely.
711
823
 
712
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
713
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
714
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
824
+ Parameters
825
+ ----------
826
+ gpu : int
827
+ Number of GPUs to use.
828
+ gpu_type : str
829
+ Type of Nvidia GPU to use.
830
+ queue_timeout : int
831
+ Time to keep the job in NVCF's queue.
832
+ """
833
+ ...
834
+
835
+ @typing.overload
836
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
837
+ """
838
+ Specifies the number of times the task corresponding
839
+ to a step needs to be retried.
715
840
 
716
- Note that all the values specified in parameters are added together so if you specify
717
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
841
+ This decorator is useful for handling transient errors, such as networking issues.
842
+ If your task contains operations that can't be retried safely, e.g. database updates,
843
+ it is advisable to annotate it with `@retry(times=0)`.
844
+
845
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
846
+ decorator will execute a no-op task after all retries have been exhausted,
847
+ ensuring that the flow execution can continue.
718
848
 
719
849
 
720
850
  Parameters
721
851
  ----------
722
- seconds : int, default 0
723
- Number of seconds to wait prior to timing out.
724
- minutes : int, default 0
725
- Number of minutes to wait prior to timing out.
726
- hours : int, default 0
727
- Number of hours to wait prior to timing out.
852
+ times : int, default 3
853
+ Number of times to retry this task.
854
+ minutes_between_retries : int, default 2
855
+ Number of minutes between retries.
728
856
  """
729
857
  ...
730
858
 
731
859
  @typing.overload
732
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
860
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
733
861
  ...
734
862
 
735
863
  @typing.overload
736
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
864
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
737
865
  ...
738
866
 
739
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
867
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
740
868
  """
741
- Specifies a timeout for your step.
869
+ Specifies the number of times the task corresponding
870
+ to a step needs to be retried.
742
871
 
743
- This decorator is useful if this step may hang indefinitely.
872
+ This decorator is useful for handling transient errors, such as networking issues.
873
+ If your task contains operations that can't be retried safely, e.g. database updates,
874
+ it is advisable to annotate it with `@retry(times=0)`.
744
875
 
745
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
746
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
747
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
748
-
749
- Note that all the values specified in parameters are added together so if you specify
750
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
751
-
752
-
753
- Parameters
754
- ----------
755
- seconds : int, default 0
756
- Number of seconds to wait prior to timing out.
757
- minutes : int, default 0
758
- Number of minutes to wait prior to timing out.
759
- hours : int, default 0
760
- Number of hours to wait prior to timing out.
761
- """
762
- ...
763
-
764
- @typing.overload
765
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
766
- """
767
- Specifies the resources needed when executing this step.
768
-
769
- Use `@resources` to specify the resource requirements
770
- independently of the specific compute layer (`@batch`, `@kubernetes`).
771
-
772
- You can choose the compute layer on the command line by executing e.g.
773
- ```
774
- python myflow.py run --with batch
775
- ```
776
- or
777
- ```
778
- python myflow.py run --with kubernetes
779
- ```
780
- which executes the flow on the desired system using the
781
- requirements specified in `@resources`.
876
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
877
+ decorator will execute a no-op task after all retries have been exhausted,
878
+ ensuring that the flow execution can continue.
782
879
 
783
880
 
784
881
  Parameters
785
882
  ----------
786
- cpu : int, default 1
787
- Number of CPUs required for this step.
788
- gpu : int, optional, default None
789
- Number of GPUs required for this step.
790
- disk : int, optional, default None
791
- Disk size (in MB) required for this step. Only applies on Kubernetes.
792
- memory : int, default 4096
793
- Memory size (in MB) required for this step.
794
- shared_memory : int, optional, default None
795
- The value for the size (in MiB) of the /dev/shm volume for this step.
796
- This parameter maps to the `--shm-size` option in Docker.
883
+ times : int, default 3
884
+ Number of times to retry this task.
885
+ minutes_between_retries : int, default 2
886
+ Number of minutes between retries.
797
887
  """
798
888
  ...
799
889
 
800
890
  @typing.overload
801
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
802
- ...
803
-
804
- @typing.overload
805
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
806
- ...
807
-
808
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
891
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
809
892
  """
810
- Specifies the resources needed when executing this step.
811
-
812
- Use `@resources` to specify the resource requirements
813
- independently of the specific compute layer (`@batch`, `@kubernetes`).
893
+ Specifies a timeout for your step.
814
894
 
815
- You can choose the compute layer on the command line by executing e.g.
816
- ```
817
- python myflow.py run --with batch
818
- ```
819
- or
820
- ```
821
- python myflow.py run --with kubernetes
822
- ```
823
- which executes the flow on the desired system using the
824
- requirements specified in `@resources`.
895
+ This decorator is useful if this step may hang indefinitely.
825
896
 
897
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
898
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
899
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
826
900
 
827
- Parameters
828
- ----------
829
- cpu : int, default 1
830
- Number of CPUs required for this step.
831
- gpu : int, optional, default None
832
- Number of GPUs required for this step.
833
- disk : int, optional, default None
834
- Disk size (in MB) required for this step. Only applies on Kubernetes.
835
- memory : int, default 4096
836
- Memory size (in MB) required for this step.
837
- shared_memory : int, optional, default None
838
- The value for the size (in MiB) of the /dev/shm volume for this step.
839
- This parameter maps to the `--shm-size` option in Docker.
840
- """
841
- ...
842
-
843
- @typing.overload
844
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
845
- """
846
- Specifies environment variables to be set prior to the execution of a step.
901
+ Note that all the values specified in parameters are added together so if you specify
902
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
847
903
 
848
904
 
849
905
  Parameters
850
906
  ----------
851
- vars : Dict[str, str], default {}
852
- Dictionary of environment variables to set.
907
+ seconds : int, default 0
908
+ Number of seconds to wait prior to timing out.
909
+ minutes : int, default 0
910
+ Number of minutes to wait prior to timing out.
911
+ hours : int, default 0
912
+ Number of hours to wait prior to timing out.
853
913
  """
854
914
  ...
855
915
 
856
916
  @typing.overload
857
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
917
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
858
918
  ...
859
919
 
860
920
  @typing.overload
861
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
862
- ...
863
-
864
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
865
- """
866
- Specifies environment variables to be set prior to the execution of a step.
867
-
868
-
869
- Parameters
870
- ----------
871
- vars : Dict[str, str], default {}
872
- Dictionary of environment variables to set.
873
- """
921
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
874
922
  ...
875
923
 
876
- @typing.overload
877
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
924
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
878
925
  """
879
- Specifies the PyPI packages for the step.
880
-
881
- Information in this decorator will augment any
882
- attributes set in the `@pyi_base` flow-level decorator. Hence,
883
- you can use `@pypi_base` to set packages required by all
884
- steps and use `@pypi` to specify step-specific overrides.
926
+ Specifies a timeout for your step.
885
927
 
928
+ This decorator is useful if this step may hang indefinitely.
886
929
 
887
- Parameters
888
- ----------
889
- packages : Dict[str, str], default: {}
890
- Packages to use for this step. The key is the name of the package
891
- and the value is the version to use.
892
- python : str, optional, default: None
893
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
894
- that the version used will correspond to the version of the Python interpreter used to start the run.
895
- """
896
- ...
897
-
898
- @typing.overload
899
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
900
- ...
901
-
902
- @typing.overload
903
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
904
- ...
905
-
906
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
907
- """
908
- Specifies the PyPI packages for the step.
930
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
931
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
932
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
909
933
 
910
- Information in this decorator will augment any
911
- attributes set in the `@pyi_base` flow-level decorator. Hence,
912
- you can use `@pypi_base` to set packages required by all
913
- steps and use `@pypi` to specify step-specific overrides.
934
+ Note that all the values specified in parameters are added together so if you specify
935
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
914
936
 
915
937
 
916
938
  Parameters
917
939
  ----------
918
- packages : Dict[str, str], default: {}
919
- Packages to use for this step. The key is the name of the package
920
- and the value is the version to use.
921
- python : str, optional, default: None
922
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
923
- that the version used will correspond to the version of the Python interpreter used to start the run.
940
+ seconds : int, default 0
941
+ Number of seconds to wait prior to timing out.
942
+ minutes : int, default 0
943
+ Number of minutes to wait prior to timing out.
944
+ hours : int, default 0
945
+ Number of hours to wait prior to timing out.
924
946
  """
925
947
  ...
926
948
 
927
949
  @typing.overload
928
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
950
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
929
951
  """
930
- Enables loading / saving of models within a step.
931
-
952
+ Specifies secrets to be retrieved and injected as environment variables prior to
953
+ the execution of a step.
932
954
 
933
955
 
934
956
  Parameters
935
957
  ----------
936
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
937
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
938
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
939
- - `current.checkpoint`
940
- - `current.model`
941
- - `current.huggingface_hub`
942
-
943
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
944
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
945
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
946
-
947
- temp_dir_root : str, default: None
948
- The root directory under which `current.model.loaded` will store loaded models
958
+ sources : List[Union[str, Dict[str, Any]]], default: []
959
+ List of secret specs, defining how the secrets are to be retrieved
949
960
  """
950
961
  ...
951
962
 
952
963
  @typing.overload
953
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
964
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
954
965
  ...
955
966
 
956
967
  @typing.overload
957
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
958
- ...
959
-
960
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
961
- """
962
- Enables loading / saving of models within a step.
963
-
964
-
965
-
966
- Parameters
967
- ----------
968
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
969
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
970
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
971
- - `current.checkpoint`
972
- - `current.model`
973
- - `current.huggingface_hub`
974
-
975
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
976
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
977
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
978
-
979
- temp_dir_root : str, default: None
980
- The root directory under which `current.model.loaded` will store loaded models
981
- """
968
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
982
969
  ...
983
970
 
984
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
971
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
985
972
  """
986
- Specifies what flows belong to the same project.
987
-
988
- A project-specific namespace is created for all flows that
989
- use the same `@project(name)`.
973
+ Specifies secrets to be retrieved and injected as environment variables prior to
974
+ the execution of a step.
990
975
 
991
976
 
992
977
  Parameters
993
978
  ----------
994
- name : str
995
- Project name. Make sure that the name is unique amongst all
996
- projects that use the same production scheduler. The name may
997
- contain only lowercase alphanumeric characters and underscores.
998
-
999
- branch : Optional[str], default None
1000
- The branch to use. If not specified, the branch is set to
1001
- `user.<username>` unless `production` is set to `True`. This can
1002
- also be set on the command line using `--branch` as a top-level option.
1003
- It is an error to specify `branch` in the decorator and on the command line.
1004
-
1005
- production : bool, default False
1006
- Whether or not the branch is the production branch. This can also be set on the
1007
- command line using `--production` as a top-level option. It is an error to specify
1008
- `production` in the decorator and on the command line.
1009
- The project branch name will be:
1010
- - if `branch` is specified:
1011
- - if `production` is True: `prod.<branch>`
1012
- - if `production` is False: `test.<branch>`
1013
- - if `branch` is not specified:
1014
- - if `production` is True: `prod`
1015
- - if `production` is False: `user.<username>`
979
+ sources : List[Union[str, Dict[str, Any]]], default: []
980
+ List of secret specs, defining how the secrets are to be retrieved
1016
981
  """
1017
982
  ...
1018
983
 
@@ -1059,107 +1024,6 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1059
1024
  """
1060
1025
  ...
1061
1026
 
1062
- @typing.overload
1063
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1064
- """
1065
- Specifies the flow(s) that this flow depends on.
1066
-
1067
- ```
1068
- @trigger_on_finish(flow='FooFlow')
1069
- ```
1070
- or
1071
- ```
1072
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1073
- ```
1074
- This decorator respects the @project decorator and triggers the flow
1075
- when upstream runs within the same namespace complete successfully
1076
-
1077
- Additionally, you can specify project aware upstream flow dependencies
1078
- by specifying the fully qualified project_flow_name.
1079
- ```
1080
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1081
- ```
1082
- or
1083
- ```
1084
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1085
- ```
1086
-
1087
- You can also specify just the project or project branch (other values will be
1088
- inferred from the current project or project branch):
1089
- ```
1090
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1091
- ```
1092
-
1093
- Note that `branch` is typically one of:
1094
- - `prod`
1095
- - `user.bob`
1096
- - `test.my_experiment`
1097
- - `prod.staging`
1098
-
1099
-
1100
- Parameters
1101
- ----------
1102
- flow : Union[str, Dict[str, str]], optional, default None
1103
- Upstream flow dependency for this flow.
1104
- flows : List[Union[str, Dict[str, str]]], default []
1105
- Upstream flow dependencies for this flow.
1106
- options : Dict[str, Any], default {}
1107
- Backend-specific configuration for tuning eventing behavior.
1108
- """
1109
- ...
1110
-
1111
- @typing.overload
1112
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1113
- ...
1114
-
1115
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1116
- """
1117
- Specifies the flow(s) that this flow depends on.
1118
-
1119
- ```
1120
- @trigger_on_finish(flow='FooFlow')
1121
- ```
1122
- or
1123
- ```
1124
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1125
- ```
1126
- This decorator respects the @project decorator and triggers the flow
1127
- when upstream runs within the same namespace complete successfully
1128
-
1129
- Additionally, you can specify project aware upstream flow dependencies
1130
- by specifying the fully qualified project_flow_name.
1131
- ```
1132
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1133
- ```
1134
- or
1135
- ```
1136
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1137
- ```
1138
-
1139
- You can also specify just the project or project branch (other values will be
1140
- inferred from the current project or project branch):
1141
- ```
1142
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1143
- ```
1144
-
1145
- Note that `branch` is typically one of:
1146
- - `prod`
1147
- - `user.bob`
1148
- - `test.my_experiment`
1149
- - `prod.staging`
1150
-
1151
-
1152
- Parameters
1153
- ----------
1154
- flow : Union[str, Dict[str, str]], optional, default None
1155
- Upstream flow dependency for this flow.
1156
- flows : List[Union[str, Dict[str, str]]], default []
1157
- Upstream flow dependencies for this flow.
1158
- options : Dict[str, Any], default {}
1159
- Backend-specific configuration for tuning eventing behavior.
1160
- """
1161
- ...
1162
-
1163
1027
  @typing.overload
1164
1028
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1165
1029
  """
@@ -1212,94 +1076,95 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1212
1076
  ...
1213
1077
 
1214
1078
  @typing.overload
1215
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1079
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1216
1080
  """
1217
- Specifies the times when the flow should be run when running on a
1218
- production scheduler.
1081
+ Specifies the event(s) that this flow depends on.
1082
+
1083
+ ```
1084
+ @trigger(event='foo')
1085
+ ```
1086
+ or
1087
+ ```
1088
+ @trigger(events=['foo', 'bar'])
1089
+ ```
1090
+
1091
+ Additionally, you can specify the parameter mappings
1092
+ to map event payload to Metaflow parameters for the flow.
1093
+ ```
1094
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1095
+ ```
1096
+ or
1097
+ ```
1098
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1099
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1100
+ ```
1101
+
1102
+ 'parameters' can also be a list of strings and tuples like so:
1103
+ ```
1104
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1105
+ ```
1106
+ This is equivalent to:
1107
+ ```
1108
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1109
+ ```
1219
1110
 
1220
1111
 
1221
1112
  Parameters
1222
1113
  ----------
1223
- hourly : bool, default False
1224
- Run the workflow hourly.
1225
- daily : bool, default True
1226
- Run the workflow daily.
1227
- weekly : bool, default False
1228
- Run the workflow weekly.
1229
- cron : str, optional, default None
1230
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1231
- specified by this expression.
1232
- timezone : str, optional, default None
1233
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1234
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1114
+ event : Union[str, Dict[str, Any]], optional, default None
1115
+ Event dependency for this flow.
1116
+ events : List[Union[str, Dict[str, Any]]], default []
1117
+ Events dependency for this flow.
1118
+ options : Dict[str, Any], default {}
1119
+ Backend-specific configuration for tuning eventing behavior.
1235
1120
  """
1236
1121
  ...
1237
1122
 
1238
1123
  @typing.overload
1239
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1124
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1240
1125
  ...
1241
1126
 
1242
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1127
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1243
1128
  """
1244
- Specifies the times when the flow should be run when running on a
1245
- production scheduler.
1246
-
1129
+ Specifies the event(s) that this flow depends on.
1247
1130
 
1248
- Parameters
1249
- ----------
1250
- hourly : bool, default False
1251
- Run the workflow hourly.
1252
- daily : bool, default True
1253
- Run the workflow daily.
1254
- weekly : bool, default False
1255
- Run the workflow weekly.
1256
- cron : str, optional, default None
1257
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1258
- specified by this expression.
1259
- timezone : str, optional, default None
1260
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1261
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1262
- """
1263
- ...
1264
-
1265
- @typing.overload
1266
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1267
- """
1268
- Specifies the PyPI packages for all steps of the flow.
1131
+ ```
1132
+ @trigger(event='foo')
1133
+ ```
1134
+ or
1135
+ ```
1136
+ @trigger(events=['foo', 'bar'])
1137
+ ```
1269
1138
 
1270
- Use `@pypi_base` to set common packages required by all
1271
- steps and use `@pypi` to specify step-specific overrides.
1139
+ Additionally, you can specify the parameter mappings
1140
+ to map event payload to Metaflow parameters for the flow.
1141
+ ```
1142
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1143
+ ```
1144
+ or
1145
+ ```
1146
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1147
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1148
+ ```
1272
1149
 
1273
- Parameters
1274
- ----------
1275
- packages : Dict[str, str], default: {}
1276
- Packages to use for this flow. The key is the name of the package
1277
- and the value is the version to use.
1278
- python : str, optional, default: None
1279
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1280
- that the version used will correspond to the version of the Python interpreter used to start the run.
1281
- """
1282
- ...
1283
-
1284
- @typing.overload
1285
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1286
- ...
1287
-
1288
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1289
- """
1290
- Specifies the PyPI packages for all steps of the flow.
1150
+ 'parameters' can also be a list of strings and tuples like so:
1151
+ ```
1152
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1153
+ ```
1154
+ This is equivalent to:
1155
+ ```
1156
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1157
+ ```
1291
1158
 
1292
- Use `@pypi_base` to set common packages required by all
1293
- steps and use `@pypi` to specify step-specific overrides.
1294
1159
 
1295
1160
  Parameters
1296
1161
  ----------
1297
- packages : Dict[str, str], default: {}
1298
- Packages to use for this flow. The key is the name of the package
1299
- and the value is the version to use.
1300
- python : str, optional, default: None
1301
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1302
- that the version used will correspond to the version of the Python interpreter used to start the run.
1162
+ event : Union[str, Dict[str, Any]], optional, default None
1163
+ Event dependency for this flow.
1164
+ events : List[Union[str, Dict[str, Any]]], default []
1165
+ Events dependency for this flow.
1166
+ options : Dict[str, Any], default {}
1167
+ Backend-specific configuration for tuning eventing behavior.
1303
1168
  """
1304
1169
  ...
1305
1170
 
@@ -1346,99 +1211,193 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1346
1211
  """
1347
1212
  ...
1348
1213
 
1214
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1215
+ """
1216
+ Specifies what flows belong to the same project.
1217
+
1218
+ A project-specific namespace is created for all flows that
1219
+ use the same `@project(name)`.
1220
+
1221
+
1222
+ Parameters
1223
+ ----------
1224
+ name : str
1225
+ Project name. Make sure that the name is unique amongst all
1226
+ projects that use the same production scheduler. The name may
1227
+ contain only lowercase alphanumeric characters and underscores.
1228
+
1229
+ branch : Optional[str], default None
1230
+ The branch to use. If not specified, the branch is set to
1231
+ `user.<username>` unless `production` is set to `True`. This can
1232
+ also be set on the command line using `--branch` as a top-level option.
1233
+ It is an error to specify `branch` in the decorator and on the command line.
1234
+
1235
+ production : bool, default False
1236
+ Whether or not the branch is the production branch. This can also be set on the
1237
+ command line using `--production` as a top-level option. It is an error to specify
1238
+ `production` in the decorator and on the command line.
1239
+ The project branch name will be:
1240
+ - if `branch` is specified:
1241
+ - if `production` is True: `prod.<branch>`
1242
+ - if `production` is False: `test.<branch>`
1243
+ - if `branch` is not specified:
1244
+ - if `production` is True: `prod`
1245
+ - if `production` is False: `user.<username>`
1246
+ """
1247
+ ...
1248
+
1349
1249
  @typing.overload
1350
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1250
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1351
1251
  """
1352
- Specifies the event(s) that this flow depends on.
1252
+ Specifies the flow(s) that this flow depends on.
1353
1253
 
1354
1254
  ```
1355
- @trigger(event='foo')
1255
+ @trigger_on_finish(flow='FooFlow')
1356
1256
  ```
1357
1257
  or
1358
1258
  ```
1359
- @trigger(events=['foo', 'bar'])
1259
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1360
1260
  ```
1261
+ This decorator respects the @project decorator and triggers the flow
1262
+ when upstream runs within the same namespace complete successfully
1361
1263
 
1362
- Additionally, you can specify the parameter mappings
1363
- to map event payload to Metaflow parameters for the flow.
1264
+ Additionally, you can specify project aware upstream flow dependencies
1265
+ by specifying the fully qualified project_flow_name.
1364
1266
  ```
1365
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1267
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1366
1268
  ```
1367
1269
  or
1368
1270
  ```
1369
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1370
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1271
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1371
1272
  ```
1372
1273
 
1373
- 'parameters' can also be a list of strings and tuples like so:
1374
- ```
1375
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1376
- ```
1377
- This is equivalent to:
1274
+ You can also specify just the project or project branch (other values will be
1275
+ inferred from the current project or project branch):
1378
1276
  ```
1379
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1277
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1380
1278
  ```
1381
1279
 
1280
+ Note that `branch` is typically one of:
1281
+ - `prod`
1282
+ - `user.bob`
1283
+ - `test.my_experiment`
1284
+ - `prod.staging`
1285
+
1382
1286
 
1383
1287
  Parameters
1384
1288
  ----------
1385
- event : Union[str, Dict[str, Any]], optional, default None
1386
- Event dependency for this flow.
1387
- events : List[Union[str, Dict[str, Any]]], default []
1388
- Events dependency for this flow.
1289
+ flow : Union[str, Dict[str, str]], optional, default None
1290
+ Upstream flow dependency for this flow.
1291
+ flows : List[Union[str, Dict[str, str]]], default []
1292
+ Upstream flow dependencies for this flow.
1389
1293
  options : Dict[str, Any], default {}
1390
1294
  Backend-specific configuration for tuning eventing behavior.
1391
1295
  """
1392
1296
  ...
1393
1297
 
1394
1298
  @typing.overload
1395
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1299
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1396
1300
  ...
1397
1301
 
1398
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1302
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1399
1303
  """
1400
- Specifies the event(s) that this flow depends on.
1304
+ Specifies the flow(s) that this flow depends on.
1401
1305
 
1402
1306
  ```
1403
- @trigger(event='foo')
1307
+ @trigger_on_finish(flow='FooFlow')
1404
1308
  ```
1405
1309
  or
1406
1310
  ```
1407
- @trigger(events=['foo', 'bar'])
1311
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1408
1312
  ```
1313
+ This decorator respects the @project decorator and triggers the flow
1314
+ when upstream runs within the same namespace complete successfully
1409
1315
 
1410
- Additionally, you can specify the parameter mappings
1411
- to map event payload to Metaflow parameters for the flow.
1316
+ Additionally, you can specify project aware upstream flow dependencies
1317
+ by specifying the fully qualified project_flow_name.
1412
1318
  ```
1413
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1319
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1414
1320
  ```
1415
1321
  or
1416
1322
  ```
1417
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1418
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1323
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1419
1324
  ```
1420
1325
 
1421
- 'parameters' can also be a list of strings and tuples like so:
1422
- ```
1423
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1424
- ```
1425
- This is equivalent to:
1326
+ You can also specify just the project or project branch (other values will be
1327
+ inferred from the current project or project branch):
1426
1328
  ```
1427
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1329
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1428
1330
  ```
1429
1331
 
1332
+ Note that `branch` is typically one of:
1333
+ - `prod`
1334
+ - `user.bob`
1335
+ - `test.my_experiment`
1336
+ - `prod.staging`
1337
+
1430
1338
 
1431
1339
  Parameters
1432
1340
  ----------
1433
- event : Union[str, Dict[str, Any]], optional, default None
1434
- Event dependency for this flow.
1435
- events : List[Union[str, Dict[str, Any]]], default []
1436
- Events dependency for this flow.
1341
+ flow : Union[str, Dict[str, str]], optional, default None
1342
+ Upstream flow dependency for this flow.
1343
+ flows : List[Union[str, Dict[str, str]]], default []
1344
+ Upstream flow dependencies for this flow.
1437
1345
  options : Dict[str, Any], default {}
1438
1346
  Backend-specific configuration for tuning eventing behavior.
1439
1347
  """
1440
1348
  ...
1441
1349
 
1350
+ @typing.overload
1351
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1352
+ """
1353
+ Specifies the times when the flow should be run when running on a
1354
+ production scheduler.
1355
+
1356
+
1357
+ Parameters
1358
+ ----------
1359
+ hourly : bool, default False
1360
+ Run the workflow hourly.
1361
+ daily : bool, default True
1362
+ Run the workflow daily.
1363
+ weekly : bool, default False
1364
+ Run the workflow weekly.
1365
+ cron : str, optional, default None
1366
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1367
+ specified by this expression.
1368
+ timezone : str, optional, default None
1369
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1370
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1371
+ """
1372
+ ...
1373
+
1374
+ @typing.overload
1375
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1376
+ ...
1377
+
1378
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1379
+ """
1380
+ Specifies the times when the flow should be run when running on a
1381
+ production scheduler.
1382
+
1383
+
1384
+ Parameters
1385
+ ----------
1386
+ hourly : bool, default False
1387
+ Run the workflow hourly.
1388
+ daily : bool, default True
1389
+ Run the workflow daily.
1390
+ weekly : bool, default False
1391
+ Run the workflow weekly.
1392
+ cron : str, optional, default None
1393
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1394
+ specified by this expression.
1395
+ timezone : str, optional, default None
1396
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1397
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1398
+ """
1399
+ ...
1400
+
1442
1401
  def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1443
1402
  """
1444
1403
  Allows setting external datastores to save data for the
@@ -1553,5 +1512,46 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1553
1512
  """
1554
1513
  ...
1555
1514
 
1515
+ @typing.overload
1516
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1517
+ """
1518
+ Specifies the PyPI packages for all steps of the flow.
1519
+
1520
+ Use `@pypi_base` to set common packages required by all
1521
+ steps and use `@pypi` to specify step-specific overrides.
1522
+
1523
+ Parameters
1524
+ ----------
1525
+ packages : Dict[str, str], default: {}
1526
+ Packages to use for this flow. The key is the name of the package
1527
+ and the value is the version to use.
1528
+ python : str, optional, default: None
1529
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1530
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1531
+ """
1532
+ ...
1533
+
1534
+ @typing.overload
1535
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1536
+ ...
1537
+
1538
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1539
+ """
1540
+ Specifies the PyPI packages for all steps of the flow.
1541
+
1542
+ Use `@pypi_base` to set common packages required by all
1543
+ steps and use `@pypi` to specify step-specific overrides.
1544
+
1545
+ Parameters
1546
+ ----------
1547
+ packages : Dict[str, str], default: {}
1548
+ Packages to use for this flow. The key is the name of the package
1549
+ and the value is the version to use.
1550
+ python : str, optional, default: None
1551
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1552
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1553
+ """
1554
+ ...
1555
+
1556
1556
  pkg_name: str
1557
1557