ob-metaflow-stubs 6.0.3.125__py2.py3-none-any.whl → 6.0.3.127__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (182) hide show
  1. metaflow-stubs/__init__.pyi +300 -290
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +9 -21
  4. metaflow-stubs/{plugins/package_cli.pyi → cli_components/__init__.pyi} +3 -3
  5. metaflow-stubs/cli_components/utils.pyi +35 -0
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +4 -4
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +12 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +18 -11
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +4 -2
  20. metaflow-stubs/metaflow_current.pyi +116 -116
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  43. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  44. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  45. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  46. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  47. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  48. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  49. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  50. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  51. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  52. metaflow-stubs/multicore_utils.pyi +2 -2
  53. metaflow-stubs/parameters.pyi +33 -16
  54. metaflow-stubs/plugins/__init__.pyi +15 -15
  55. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  56. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  57. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  58. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  59. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  60. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  61. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  62. metaflow-stubs/plugins/argo/__init__.pyi +2 -5
  63. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  64. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  65. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -2
  66. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  68. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +30 -2
  69. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  71. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  72. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  74. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  75. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  76. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  77. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  78. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -3
  79. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +3 -2
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  85. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  92. metaflow-stubs/plugins/cards/__init__.pyi +6 -7
  93. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  94. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  95. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  96. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  98. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  99. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  100. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  101. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  102. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  103. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  104. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  105. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  106. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  107. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  110. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  111. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  112. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  113. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  114. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  115. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  116. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  119. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  120. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  121. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  122. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  123. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  124. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  125. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  126. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -5
  127. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  128. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  129. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  130. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  131. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  132. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  133. metaflow-stubs/plugins/perimeters.pyi +2 -2
  134. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  135. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  136. metaflow-stubs/plugins/pypi/conda_decorator.pyi +6 -2
  137. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  138. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  139. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  140. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  141. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  142. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  143. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  144. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  145. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  146. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  147. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  148. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  149. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  150. metaflow-stubs/profilers/__init__.pyi +2 -2
  151. metaflow-stubs/pylint_wrapper.pyi +2 -2
  152. metaflow-stubs/runner/__init__.pyi +2 -2
  153. metaflow-stubs/runner/deployer.pyi +30 -30
  154. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  155. metaflow-stubs/runner/metaflow_runner.pyi +5 -5
  156. metaflow-stubs/runner/nbdeploy.pyi +4 -2
  157. metaflow-stubs/runner/nbrun.pyi +3 -3
  158. metaflow-stubs/runner/subprocess_manager.pyi +5 -2
  159. metaflow-stubs/runner/utils.pyi +7 -7
  160. metaflow-stubs/system/__init__.pyi +2 -2
  161. metaflow-stubs/system/system_logger.pyi +2 -2
  162. metaflow-stubs/system/system_monitor.pyi +2 -2
  163. metaflow-stubs/tagging_util.pyi +2 -2
  164. metaflow-stubs/tuple_util.pyi +2 -2
  165. metaflow-stubs/{plugins/aws/step_functions/production_token.pyi → user_configs/__init__.pyi} +5 -11
  166. metaflow-stubs/user_configs/config_decorators.pyi +253 -0
  167. metaflow-stubs/user_configs/config_options.pyi +82 -0
  168. metaflow-stubs/user_configs/config_parameters.pyi +217 -0
  169. {ob_metaflow_stubs-6.0.3.125.dist-info → ob_metaflow_stubs-6.0.3.127.dist-info}/METADATA +1 -1
  170. ob_metaflow_stubs-6.0.3.127.dist-info/RECORD +173 -0
  171. metaflow-stubs/clone_util.pyi +0 -14
  172. metaflow-stubs/mflog/__init__.pyi +0 -6
  173. metaflow-stubs/mflog/mflog.pyi +0 -69
  174. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +0 -103
  175. metaflow-stubs/plugins/cards/card_cli.pyi +0 -146
  176. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +0 -33
  177. metaflow-stubs/plugins/logs_cli.pyi +0 -46
  178. metaflow-stubs/plugins/tag_cli.pyi +0 -19
  179. metaflow-stubs/procpoll.pyi +0 -53
  180. ob_metaflow_stubs-6.0.3.125.dist-info/RECORD +0 -178
  181. {ob_metaflow_stubs-6.0.3.125.dist-info → ob_metaflow_stubs-6.0.3.127.dist-info}/WHEEL +0 -0
  182. {ob_metaflow_stubs-6.0.3.125.dist-info → ob_metaflow_stubs-6.0.3.127.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.36.3+obcheckpoint(0.1.4);ob(v1) #
4
- # Generated on 2024-12-10T22:06:22.196386 #
3
+ # MF version: 2.12.39.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-12-17T21:56:51.044544 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -22,6 +22,7 @@ from .multicore_utils import parallel_map as parallel_map
22
22
  from . import metaflow_current as metaflow_current
23
23
  from .metaflow_current import current as current
24
24
  from . import parameters as parameters
25
+ from . import user_configs as user_configs
25
26
  from . import tagging_util as tagging_util
26
27
  from . import metadata_provider as metadata_provider
27
28
  from . import flowspec as flowspec
@@ -29,6 +30,10 @@ from .flowspec import FlowSpec as FlowSpec
29
30
  from .parameters import Parameter as Parameter
30
31
  from .parameters import JSONTypeClass as JSONTypeClass
31
32
  from .parameters import JSONType as JSONType
33
+ from .user_configs.config_parameters import Config as Config
34
+ from .user_configs.config_parameters import config_expr as config_expr
35
+ from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
36
+ from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
32
37
  from . import tuple_util as tuple_util
33
38
  from . import cards as cards
34
39
  from . import events as events
@@ -59,10 +64,9 @@ from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.
59
64
  from .mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures import load_model as load_model
60
65
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import get_aws_client as get_aws_client
61
66
  from .mf_extensions.outerbounds.plugins.snowflake.snowflake import Snowflake as Snowflake
67
+ from . import cli_components as cli_components
62
68
  from . import system as system
63
69
  from . import pylint_wrapper as pylint_wrapper
64
- from . import procpoll as procpoll
65
- from . import clone_util as clone_util
66
70
  from . import cli as cli
67
71
  from . import profilers as profilers
68
72
 
@@ -142,89 +146,129 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
142
146
  ...
143
147
 
144
148
  @typing.overload
145
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
149
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
146
150
  """
147
- Internal decorator to support Fast bakery
151
+ Creates a human-readable report, a Metaflow Card, after this step completes.
152
+
153
+ Note that you may add multiple `@card` decorators in a step with different parameters.
148
154
  """
149
155
  ...
150
156
 
151
157
  @typing.overload
152
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
158
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
153
159
  ...
154
160
 
155
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
161
+ @typing.overload
162
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
163
+ ...
164
+
165
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
156
166
  """
157
- Internal decorator to support Fast bakery
167
+ Creates a human-readable report, a Metaflow Card, after this step completes.
168
+
169
+ Note that you may add multiple `@card` decorators in a step with different parameters.
158
170
  """
159
171
  ...
160
172
 
161
173
  @typing.overload
162
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
174
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
163
175
  """
164
- Specifies environment variables to be set prior to the execution of a step.
176
+ Enables checkpointing for a step.
165
177
  """
166
178
  ...
167
179
 
168
180
  @typing.overload
169
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
181
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
170
182
  ...
171
183
 
172
184
  @typing.overload
173
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
185
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
174
186
  ...
175
187
 
176
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
188
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
177
189
  """
178
- Specifies environment variables to be set prior to the execution of a step.
190
+ Enables checkpointing for a step.
179
191
  """
180
192
  ...
181
193
 
182
194
  @typing.overload
183
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
195
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
184
196
  """
185
- Specifies the resources needed when executing this step.
197
+ Decorator prototype for all step decorators. This function gets specialized
198
+ and imported for all decorators types by _import_plugin_decorators().
199
+ """
200
+ ...
201
+
202
+ @typing.overload
203
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
204
+ ...
205
+
206
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
207
+ """
208
+ Decorator prototype for all step decorators. This function gets specialized
209
+ and imported for all decorators types by _import_plugin_decorators().
210
+ """
211
+ ...
212
+
213
+ @typing.overload
214
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
215
+ """
216
+ Specifies the number of times the task corresponding
217
+ to a step needs to be retried.
186
218
 
187
- Use `@resources` to specify the resource requirements
188
- independently of the specific compute layer (`@batch`, `@kubernetes`).
219
+ This decorator is useful for handling transient errors, such as networking issues.
220
+ If your task contains operations that can't be retried safely, e.g. database updates,
221
+ it is advisable to annotate it with `@retry(times=0)`.
189
222
 
190
- You can choose the compute layer on the command line by executing e.g.
191
- ```
192
- python myflow.py run --with batch
193
- ```
194
- or
195
- ```
196
- python myflow.py run --with kubernetes
197
- ```
198
- which executes the flow on the desired system using the
199
- requirements specified in `@resources`.
223
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
224
+ decorator will execute a no-op task after all retries have been exhausted,
225
+ ensuring that the flow execution can continue.
200
226
  """
201
227
  ...
202
228
 
203
229
  @typing.overload
204
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
230
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
205
231
  ...
206
232
 
207
233
  @typing.overload
208
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
234
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
209
235
  ...
210
236
 
211
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
237
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
212
238
  """
213
- Specifies the resources needed when executing this step.
239
+ Specifies the number of times the task corresponding
240
+ to a step needs to be retried.
214
241
 
215
- Use `@resources` to specify the resource requirements
216
- independently of the specific compute layer (`@batch`, `@kubernetes`).
242
+ This decorator is useful for handling transient errors, such as networking issues.
243
+ If your task contains operations that can't be retried safely, e.g. database updates,
244
+ it is advisable to annotate it with `@retry(times=0)`.
217
245
 
218
- You can choose the compute layer on the command line by executing e.g.
219
- ```
220
- python myflow.py run --with batch
221
- ```
222
- or
223
- ```
224
- python myflow.py run --with kubernetes
225
- ```
226
- which executes the flow on the desired system using the
227
- requirements specified in `@resources`.
246
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
247
+ decorator will execute a no-op task after all retries have been exhausted,
248
+ ensuring that the flow execution can continue.
249
+ """
250
+ ...
251
+
252
+ @typing.overload
253
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
254
+ """
255
+ Specifies secrets to be retrieved and injected as environment variables prior to
256
+ the execution of a step.
257
+ """
258
+ ...
259
+
260
+ @typing.overload
261
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
262
+ ...
263
+
264
+ @typing.overload
265
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
266
+ ...
267
+
268
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
269
+ """
270
+ Specifies secrets to be retrieved and injected as environment variables prior to
271
+ the execution of a step.
228
272
  """
229
273
  ...
230
274
 
@@ -260,23 +304,51 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
260
304
  ...
261
305
 
262
306
  @typing.overload
263
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
307
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
264
308
  """
265
- Enables loading / saving of models within a step.
309
+ Specifies the resources needed when executing this step.
310
+
311
+ Use `@resources` to specify the resource requirements
312
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
313
+
314
+ You can choose the compute layer on the command line by executing e.g.
315
+ ```
316
+ python myflow.py run --with batch
317
+ ```
318
+ or
319
+ ```
320
+ python myflow.py run --with kubernetes
321
+ ```
322
+ which executes the flow on the desired system using the
323
+ requirements specified in `@resources`.
266
324
  """
267
325
  ...
268
326
 
269
327
  @typing.overload
270
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
328
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
271
329
  ...
272
330
 
273
331
  @typing.overload
274
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
332
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
275
333
  ...
276
334
 
277
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
335
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
278
336
  """
279
- Enables loading / saving of models within a step.
337
+ Specifies the resources needed when executing this step.
338
+
339
+ Use `@resources` to specify the resource requirements
340
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
341
+
342
+ You can choose the compute layer on the command line by executing e.g.
343
+ ```
344
+ python myflow.py run --with batch
345
+ ```
346
+ or
347
+ ```
348
+ python myflow.py run --with kubernetes
349
+ ```
350
+ which executes the flow on the desired system using the
351
+ requirements specified in `@resources`.
280
352
  """
281
353
  ...
282
354
 
@@ -343,46 +415,79 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
343
415
  ...
344
416
 
345
417
  @typing.overload
346
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
418
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
347
419
  """
348
- Creates a human-readable report, a Metaflow Card, after this step completes.
349
-
350
- Note that you may add multiple `@card` decorators in a step with different parameters.
420
+ Enables loading / saving of models within a step.
351
421
  """
352
422
  ...
353
423
 
354
424
  @typing.overload
355
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
425
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
356
426
  ...
357
427
 
358
428
  @typing.overload
359
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
429
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
360
430
  ...
361
431
 
362
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
432
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
363
433
  """
364
- Creates a human-readable report, a Metaflow Card, after this step completes.
365
-
366
- Note that you may add multiple `@card` decorators in a step with different parameters.
434
+ Enables loading / saving of models within a step.
367
435
  """
368
436
  ...
369
437
 
370
438
  @typing.overload
371
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
439
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
372
440
  """
373
- Decorator prototype for all step decorators. This function gets specialized
374
- and imported for all decorators types by _import_plugin_decorators().
441
+ Specifies environment variables to be set prior to the execution of a step.
375
442
  """
376
443
  ...
377
444
 
378
445
  @typing.overload
379
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
446
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
380
447
  ...
381
448
 
382
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
449
+ @typing.overload
450
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
451
+ ...
452
+
453
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
383
454
  """
384
- Decorator prototype for all step decorators. This function gets specialized
385
- and imported for all decorators types by _import_plugin_decorators().
455
+ Specifies environment variables to be set prior to the execution of a step.
456
+ """
457
+ ...
458
+
459
+ def nvidia(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
460
+ """
461
+ Specifies that this step should execute on DGX cloud.
462
+ """
463
+ ...
464
+
465
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
466
+ """
467
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
468
+ """
469
+ ...
470
+
471
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
472
+ """
473
+ Specifies that this step should execute on Kubernetes.
474
+ """
475
+ ...
476
+
477
+ @typing.overload
478
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
479
+ """
480
+ Internal decorator to support Fast bakery
481
+ """
482
+ ...
483
+
484
+ @typing.overload
485
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
486
+ ...
487
+
488
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
489
+ """
490
+ Internal decorator to support Fast bakery
386
491
  """
387
492
  ...
388
493
 
@@ -425,139 +530,114 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
425
530
  """
426
531
  ...
427
532
 
428
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
533
+ @typing.overload
534
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
429
535
  """
430
- Decorator that helps cache, version and store models/datasets from huggingface hub.
536
+ Specifies the Conda environment for all steps of the flow.
537
+
538
+ Use `@conda_base` to set common libraries required by all
539
+ steps and use `@conda` to specify step-specific additions.
431
540
  """
432
541
  ...
433
542
 
434
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
543
+ @typing.overload
544
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
545
+ ...
546
+
547
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
435
548
  """
436
- Specifies that this step should execute on Kubernetes.
549
+ Specifies the Conda environment for all steps of the flow.
550
+
551
+ Use `@conda_base` to set common libraries required by all
552
+ steps and use `@conda` to specify step-specific additions.
437
553
  """
438
554
  ...
439
555
 
440
- @typing.overload
441
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
556
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
442
557
  """
443
- Specifies secrets to be retrieved and injected as environment variables prior to
444
- the execution of a step.
558
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
559
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
445
560
  """
446
561
  ...
447
562
 
448
563
  @typing.overload
449
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
450
- ...
451
-
452
- @typing.overload
453
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
454
- ...
455
-
456
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
457
- """
458
- Specifies secrets to be retrieved and injected as environment variables prior to
459
- the execution of a step.
460
- """
461
- ...
462
-
463
- @typing.overload
464
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
465
- """
466
- Enables checkpointing for a step.
467
- """
468
- ...
469
-
470
- @typing.overload
471
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
472
- ...
473
-
474
- @typing.overload
475
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
476
- ...
477
-
478
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
479
- """
480
- Enables checkpointing for a step.
481
- """
482
- ...
483
-
484
- @typing.overload
485
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
564
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
486
565
  """
487
- Specifies the number of times the task corresponding
488
- to a step needs to be retried.
566
+ Specifies the flow(s) that this flow depends on.
489
567
 
490
- This decorator is useful for handling transient errors, such as networking issues.
491
- If your task contains operations that can't be retried safely, e.g. database updates,
492
- it is advisable to annotate it with `@retry(times=0)`.
568
+ ```
569
+ @trigger_on_finish(flow='FooFlow')
570
+ ```
571
+ or
572
+ ```
573
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
574
+ ```
575
+ This decorator respects the @project decorator and triggers the flow
576
+ when upstream runs within the same namespace complete successfully
493
577
 
494
- This can be used in conjunction with the `@catch` decorator. The `@catch`
495
- decorator will execute a no-op task after all retries have been exhausted,
496
- ensuring that the flow execution can continue.
578
+ Additionally, you can specify project aware upstream flow dependencies
579
+ by specifying the fully qualified project_flow_name.
580
+ ```
581
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
582
+ ```
583
+ or
584
+ ```
585
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
586
+ ```
587
+
588
+ You can also specify just the project or project branch (other values will be
589
+ inferred from the current project or project branch):
590
+ ```
591
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
592
+ ```
593
+
594
+ Note that `branch` is typically one of:
595
+ - `prod`
596
+ - `user.bob`
597
+ - `test.my_experiment`
598
+ - `prod.staging`
497
599
  """
498
600
  ...
499
601
 
500
602
  @typing.overload
501
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
502
- ...
503
-
504
- @typing.overload
505
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
603
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
506
604
  ...
507
605
 
508
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
606
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
509
607
  """
510
- Specifies the number of times the task corresponding
511
- to a step needs to be retried.
512
-
513
- This decorator is useful for handling transient errors, such as networking issues.
514
- If your task contains operations that can't be retried safely, e.g. database updates,
515
- it is advisable to annotate it with `@retry(times=0)`.
608
+ Specifies the flow(s) that this flow depends on.
516
609
 
517
- This can be used in conjunction with the `@catch` decorator. The `@catch`
518
- decorator will execute a no-op task after all retries have been exhausted,
519
- ensuring that the flow execution can continue.
520
- """
521
- ...
522
-
523
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
524
- """
525
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
610
+ ```
611
+ @trigger_on_finish(flow='FooFlow')
612
+ ```
613
+ or
614
+ ```
615
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
616
+ ```
617
+ This decorator respects the @project decorator and triggers the flow
618
+ when upstream runs within the same namespace complete successfully
526
619
 
527
- User code call
528
- -----------
529
- @nim(
530
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
531
- backend='managed'
532
- )
620
+ Additionally, you can specify project aware upstream flow dependencies
621
+ by specifying the fully qualified project_flow_name.
622
+ ```
623
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
624
+ ```
625
+ or
626
+ ```
627
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
628
+ ```
533
629
 
534
- Valid backend options
535
- ---------------------
536
- - 'managed': Outerbounds selects a compute provider based on the model.
537
- - 🚧 'dataplane': Run in your account.
630
+ You can also specify just the project or project branch (other values will be
631
+ inferred from the current project or project branch):
632
+ ```
633
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
634
+ ```
538
635
 
539
- Valid model options
540
- ----------------
541
- - 'meta/llama3-8b-instruct': 8B parameter model
542
- - 'meta/llama3-70b-instruct': 70B parameter model
543
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
544
- """
545
- ...
546
-
547
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
548
- """
549
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
550
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
551
- """
552
- ...
553
-
554
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
555
- """
556
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
557
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
558
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
559
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
560
- starts only after all sensors finish.
636
+ Note that `branch` is typically one of:
637
+ - `prod`
638
+ - `user.bob`
639
+ - `test.my_experiment`
640
+ - `prod.staging`
561
641
  """
562
642
  ...
563
643
 
@@ -635,157 +715,87 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
635
715
  ...
636
716
 
637
717
  @typing.overload
638
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
718
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
639
719
  """
640
- Specifies the times when the flow should be run when running on a
641
- production scheduler.
720
+ Specifies the PyPI packages for all steps of the flow.
721
+
722
+ Use `@pypi_base` to set common packages required by all
723
+ steps and use `@pypi` to specify step-specific overrides.
642
724
  """
643
725
  ...
644
726
 
645
727
  @typing.overload
646
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
647
- ...
648
-
649
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
650
- """
651
- Specifies the times when the flow should be run when running on a
652
- production scheduler.
653
- """
728
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
654
729
  ...
655
730
 
656
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
731
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
657
732
  """
658
- Specifies what flows belong to the same project.
733
+ Specifies the PyPI packages for all steps of the flow.
659
734
 
660
- A project-specific namespace is created for all flows that
661
- use the same `@project(name)`.
735
+ Use `@pypi_base` to set common packages required by all
736
+ steps and use `@pypi` to specify step-specific overrides.
662
737
  """
663
738
  ...
664
739
 
665
- @typing.overload
666
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
740
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
667
741
  """
668
- Specifies the Conda environment for all steps of the flow.
742
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
669
743
 
670
- Use `@conda_base` to set common libraries required by all
671
- steps and use `@conda` to specify step-specific additions.
672
- """
673
- ...
674
-
675
- @typing.overload
676
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
677
- ...
678
-
679
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
680
- """
681
- Specifies the Conda environment for all steps of the flow.
744
+ User code call
745
+ -----------
746
+ @nim(
747
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
748
+ backend='managed'
749
+ )
682
750
 
683
- Use `@conda_base` to set common libraries required by all
684
- steps and use `@conda` to specify step-specific additions.
751
+ Valid backend options
752
+ ---------------------
753
+ - 'managed': Outerbounds selects a compute provider based on the model.
754
+ - 🚧 'dataplane': Run in your account.
755
+
756
+ Valid model options
757
+ ----------------
758
+ - 'meta/llama3-8b-instruct': 8B parameter model
759
+ - 'meta/llama3-70b-instruct': 70B parameter model
760
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
685
761
  """
686
762
  ...
687
763
 
688
764
  @typing.overload
689
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
765
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
690
766
  """
691
- Specifies the PyPI packages for all steps of the flow.
692
-
693
- Use `@pypi_base` to set common packages required by all
694
- steps and use `@pypi` to specify step-specific overrides.
767
+ Specifies the times when the flow should be run when running on a
768
+ production scheduler.
695
769
  """
696
770
  ...
697
771
 
698
772
  @typing.overload
699
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
773
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
700
774
  ...
701
775
 
702
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
776
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
703
777
  """
704
- Specifies the PyPI packages for all steps of the flow.
705
-
706
- Use `@pypi_base` to set common packages required by all
707
- steps and use `@pypi` to specify step-specific overrides.
778
+ Specifies the times when the flow should be run when running on a
779
+ production scheduler.
708
780
  """
709
781
  ...
710
782
 
711
- @typing.overload
712
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
783
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
713
784
  """
714
- Specifies the flow(s) that this flow depends on.
715
-
716
- ```
717
- @trigger_on_finish(flow='FooFlow')
718
- ```
719
- or
720
- ```
721
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
722
- ```
723
- This decorator respects the @project decorator and triggers the flow
724
- when upstream runs within the same namespace complete successfully
725
-
726
- Additionally, you can specify project aware upstream flow dependencies
727
- by specifying the fully qualified project_flow_name.
728
- ```
729
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
730
- ```
731
- or
732
- ```
733
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
734
- ```
735
-
736
- You can also specify just the project or project branch (other values will be
737
- inferred from the current project or project branch):
738
- ```
739
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
740
- ```
741
-
742
- Note that `branch` is typically one of:
743
- - `prod`
744
- - `user.bob`
745
- - `test.my_experiment`
746
- - `prod.staging`
785
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
786
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
787
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
788
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
789
+ starts only after all sensors finish.
747
790
  """
748
791
  ...
749
792
 
750
- @typing.overload
751
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
752
- ...
753
-
754
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
793
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
755
794
  """
756
- Specifies the flow(s) that this flow depends on.
757
-
758
- ```
759
- @trigger_on_finish(flow='FooFlow')
760
- ```
761
- or
762
- ```
763
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
764
- ```
765
- This decorator respects the @project decorator and triggers the flow
766
- when upstream runs within the same namespace complete successfully
767
-
768
- Additionally, you can specify project aware upstream flow dependencies
769
- by specifying the fully qualified project_flow_name.
770
- ```
771
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
772
- ```
773
- or
774
- ```
775
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
776
- ```
777
-
778
- You can also specify just the project or project branch (other values will be
779
- inferred from the current project or project branch):
780
- ```
781
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
782
- ```
795
+ Specifies what flows belong to the same project.
783
796
 
784
- Note that `branch` is typically one of:
785
- - `prod`
786
- - `user.bob`
787
- - `test.my_experiment`
788
- - `prod.staging`
797
+ A project-specific namespace is created for all flows that
798
+ use the same `@project(name)`.
789
799
  """
790
800
  ...
791
801