ob-metaflow-stubs 6.0.3.182rc1__py2.py3-none-any.whl → 6.0.3.182rc2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. metaflow-stubs/__init__.pyi +616 -616
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +1 -1
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +3 -3
  14. metaflow-stubs/info_file.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +34 -34
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +1 -1
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +1 -1
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +1 -1
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +1 -1
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +2 -2
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +1 -1
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +1 -1
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +1 -1
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  87. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  88. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  89. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
  90. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  91. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
  92. metaflow-stubs/multicore_utils.pyi +1 -1
  93. metaflow-stubs/ob_internal.pyi +1 -1
  94. metaflow-stubs/parameters.pyi +2 -2
  95. metaflow-stubs/plugins/__init__.pyi +13 -13
  96. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  97. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  98. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  99. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  100. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  101. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  102. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  103. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  104. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  105. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  106. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  107. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  108. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  109. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  110. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  111. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  112. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  113. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  114. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  115. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  116. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  117. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  118. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +1 -1
  119. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  120. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  121. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  122. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  123. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  124. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  125. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  126. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  127. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  128. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  129. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +1 -1
  130. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  131. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  132. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  133. metaflow-stubs/plugins/cards/__init__.pyi +5 -5
  134. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  135. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  136. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  137. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  138. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  139. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  140. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  141. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  142. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  143. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  144. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  145. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  146. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  147. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  148. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  149. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  150. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  151. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  152. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  153. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  154. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  155. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  156. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  157. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  158. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  159. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  160. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  161. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  162. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +1 -1
  163. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  164. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  165. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  166. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  167. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  168. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  169. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  170. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  171. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  172. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  173. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  174. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  175. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  176. metaflow-stubs/plugins/perimeters.pyi +1 -1
  177. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  178. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  179. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  180. metaflow-stubs/plugins/pypi/conda_environment.pyi +2 -2
  181. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  182. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  183. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  184. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  185. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  186. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  187. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  188. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
  189. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  190. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  191. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  192. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  193. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  194. metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
  195. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  196. metaflow-stubs/plugins/uv/uv_environment.pyi +1 -1
  197. metaflow-stubs/profilers/__init__.pyi +1 -1
  198. metaflow-stubs/pylint_wrapper.pyi +1 -1
  199. metaflow-stubs/runner/__init__.pyi +1 -1
  200. metaflow-stubs/runner/deployer.pyi +29 -29
  201. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  202. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  203. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  204. metaflow-stubs/runner/nbrun.pyi +1 -1
  205. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  206. metaflow-stubs/runner/utils.pyi +2 -2
  207. metaflow-stubs/system/__init__.pyi +1 -1
  208. metaflow-stubs/system/system_logger.pyi +1 -1
  209. metaflow-stubs/system/system_monitor.pyi +1 -1
  210. metaflow-stubs/tagging_util.pyi +1 -1
  211. metaflow-stubs/tuple_util.pyi +1 -1
  212. metaflow-stubs/user_configs/__init__.pyi +1 -1
  213. metaflow-stubs/user_configs/config_decorators.pyi +4 -4
  214. metaflow-stubs/user_configs/config_options.pyi +2 -2
  215. metaflow-stubs/user_configs/config_parameters.pyi +4 -4
  216. {ob_metaflow_stubs-6.0.3.182rc1.dist-info → ob_metaflow_stubs-6.0.3.182rc2.dist-info}/METADATA +1 -1
  217. ob_metaflow_stubs-6.0.3.182rc2.dist-info/RECORD +220 -0
  218. ob_metaflow_stubs-6.0.3.182rc1.dist-info/RECORD +0 -220
  219. {ob_metaflow_stubs-6.0.3.182rc1.dist-info → ob_metaflow_stubs-6.0.3.182rc2.dist-info}/WHEEL +0 -0
  220. {ob_metaflow_stubs-6.0.3.182rc1.dist-info → ob_metaflow_stubs-6.0.3.182rc2.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.15.17.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-06-18T10:06:22.722981 #
4
+ # Generated on 2025-06-18T10:30:41.738497 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -35,18 +35,18 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
- from . import metaflow_git as metaflow_git
39
38
  from . import tuple_util as tuple_util
40
39
  from . import cards as cards
41
40
  from . import events as events
41
+ from . import metaflow_git as metaflow_git
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
+ from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
47
48
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
48
49
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
49
- from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
52
52
  from .client.core import get_namespace as get_namespace
@@ -155,243 +155,37 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
155
155
  """
156
156
  ...
157
157
 
158
- @typing.overload
159
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
160
- """
161
- Specifies that the step will success under all circumstances.
162
-
163
- The decorator will create an optional artifact, specified by `var`, which
164
- contains the exception raised. You can use it to detect the presence
165
- of errors, indicating that all happy-path artifacts produced by the step
166
- are missing.
167
-
168
-
169
- Parameters
170
- ----------
171
- var : str, optional, default None
172
- Name of the artifact in which to store the caught exception.
173
- If not specified, the exception is not stored.
174
- print_exception : bool, default True
175
- Determines whether or not the exception is printed to
176
- stdout when caught.
177
- """
178
- ...
179
-
180
- @typing.overload
181
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
182
- ...
183
-
184
- @typing.overload
185
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
186
- ...
187
-
188
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
189
- """
190
- Specifies that the step will success under all circumstances.
191
-
192
- The decorator will create an optional artifact, specified by `var`, which
193
- contains the exception raised. You can use it to detect the presence
194
- of errors, indicating that all happy-path artifacts produced by the step
195
- are missing.
196
-
197
-
198
- Parameters
199
- ----------
200
- var : str, optional, default None
201
- Name of the artifact in which to store the caught exception.
202
- If not specified, the exception is not stored.
203
- print_exception : bool, default True
204
- Determines whether or not the exception is printed to
205
- stdout when caught.
206
- """
207
- ...
208
-
209
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
210
- """
211
- Specifies that this step should execute on Kubernetes.
212
-
213
-
214
- Parameters
215
- ----------
216
- cpu : int, default 1
217
- Number of CPUs required for this step. If `@resources` is
218
- also present, the maximum value from all decorators is used.
219
- memory : int, default 4096
220
- Memory size (in MB) required for this step. If
221
- `@resources` is also present, the maximum value from all decorators is
222
- used.
223
- disk : int, default 10240
224
- Disk size (in MB) required for this step. If
225
- `@resources` is also present, the maximum value from all decorators is
226
- used.
227
- image : str, optional, default None
228
- Docker image to use when launching on Kubernetes. If not specified, and
229
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
230
- not, a default Docker image mapping to the current version of Python is used.
231
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
232
- If given, the imagePullPolicy to be applied to the Docker image of the step.
233
- image_pull_secrets: List[str], default []
234
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
235
- Kubernetes image pull secrets to use when pulling container images
236
- in Kubernetes.
237
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
238
- Kubernetes service account to use when launching pod in Kubernetes.
239
- secrets : List[str], optional, default None
240
- Kubernetes secrets to use when launching pod in Kubernetes. These
241
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
242
- in Metaflow configuration.
243
- node_selector: Union[Dict[str,str], str], optional, default None
244
- Kubernetes node selector(s) to apply to the pod running the task.
245
- Can be passed in as a comma separated string of values e.g.
246
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
247
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
248
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
249
- Kubernetes namespace to use when launching pod in Kubernetes.
250
- gpu : int, optional, default None
251
- Number of GPUs required for this step. A value of zero implies that
252
- the scheduled node should not have GPUs.
253
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
254
- The vendor of the GPUs to be used for this step.
255
- tolerations : List[str], default []
256
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
257
- Kubernetes tolerations to use when launching pod in Kubernetes.
258
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
259
- Kubernetes labels to use when launching pod in Kubernetes.
260
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
261
- Kubernetes annotations to use when launching pod in Kubernetes.
262
- use_tmpfs : bool, default False
263
- This enables an explicit tmpfs mount for this step.
264
- tmpfs_tempdir : bool, default True
265
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
266
- tmpfs_size : int, optional, default: None
267
- The value for the size (in MiB) of the tmpfs mount for this step.
268
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
269
- memory allocated for this step.
270
- tmpfs_path : str, optional, default /metaflow_temp
271
- Path to tmpfs mount for this step.
272
- persistent_volume_claims : Dict[str, str], optional, default None
273
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
274
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
275
- shared_memory: int, optional
276
- Shared memory size (in MiB) required for this step
277
- port: int, optional
278
- Port number to specify in the Kubernetes job object
279
- compute_pool : str, optional, default None
280
- Compute pool to be used for for this step.
281
- If not specified, any accessible compute pool within the perimeter is used.
282
- hostname_resolution_timeout: int, default 10 * 60
283
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
284
- Only applicable when @parallel is used.
285
- qos: str, default: Burstable
286
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
287
-
288
- security_context: Dict[str, Any], optional, default None
289
- Container security context. Applies to the task container. Allows the following keys:
290
- - privileged: bool, optional, default None
291
- - allow_privilege_escalation: bool, optional, default None
292
- - run_as_user: int, optional, default None
293
- - run_as_group: int, optional, default None
294
- - run_as_non_root: bool, optional, default None
295
- """
296
- ...
297
-
298
- @typing.overload
299
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
300
- """
301
- Creates a human-readable report, a Metaflow Card, after this step completes.
302
-
303
- Note that you may add multiple `@card` decorators in a step with different parameters.
304
-
305
-
306
- Parameters
307
- ----------
308
- type : str, default 'default'
309
- Card type.
310
- id : str, optional, default None
311
- If multiple cards are present, use this id to identify this card.
312
- options : Dict[str, Any], default {}
313
- Options passed to the card. The contents depend on the card type.
314
- timeout : int, default 45
315
- Interrupt reporting if it takes more than this many seconds.
316
- """
317
- ...
318
-
319
- @typing.overload
320
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
321
- ...
322
-
323
- @typing.overload
324
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
325
- ...
326
-
327
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
158
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
328
159
  """
329
- Creates a human-readable report, a Metaflow Card, after this step completes.
330
-
331
- Note that you may add multiple `@card` decorators in a step with different parameters.
160
+ Specifies that this step is used to deploy an instance of the app.
161
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
332
162
 
333
163
 
334
164
  Parameters
335
165
  ----------
336
- type : str, default 'default'
337
- Card type.
338
- id : str, optional, default None
339
- If multiple cards are present, use this id to identify this card.
340
- options : Dict[str, Any], default {}
341
- Options passed to the card. The contents depend on the card type.
342
- timeout : int, default 45
343
- Interrupt reporting if it takes more than this many seconds.
166
+ app_port : int
167
+ Number of GPUs to use.
168
+ app_name : str
169
+ Name of the app to deploy.
344
170
  """
345
171
  ...
346
172
 
347
173
  @typing.overload
348
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
174
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
349
175
  """
350
- Specifies the PyPI packages for the step.
351
-
352
- Information in this decorator will augment any
353
- attributes set in the `@pyi_base` flow-level decorator. Hence,
354
- you can use `@pypi_base` to set packages required by all
355
- steps and use `@pypi` to specify step-specific overrides.
356
-
357
-
358
- Parameters
359
- ----------
360
- packages : Dict[str, str], default: {}
361
- Packages to use for this step. The key is the name of the package
362
- and the value is the version to use.
363
- python : str, optional, default: None
364
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
365
- that the version used will correspond to the version of the Python interpreter used to start the run.
176
+ Decorator prototype for all step decorators. This function gets specialized
177
+ and imported for all decorators types by _import_plugin_decorators().
366
178
  """
367
179
  ...
368
180
 
369
181
  @typing.overload
370
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
371
- ...
372
-
373
- @typing.overload
374
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
182
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
375
183
  ...
376
184
 
377
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
185
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
378
186
  """
379
- Specifies the PyPI packages for the step.
380
-
381
- Information in this decorator will augment any
382
- attributes set in the `@pyi_base` flow-level decorator. Hence,
383
- you can use `@pypi_base` to set packages required by all
384
- steps and use `@pypi` to specify step-specific overrides.
385
-
386
-
387
- Parameters
388
- ----------
389
- packages : Dict[str, str], default: {}
390
- Packages to use for this step. The key is the name of the package
391
- and the value is the version to use.
392
- python : str, optional, default: None
393
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
394
- that the version used will correspond to the version of the Python interpreter used to start the run.
187
+ Decorator prototype for all step decorators. This function gets specialized
188
+ and imported for all decorators types by _import_plugin_decorators().
395
189
  """
396
190
  ...
397
191
 
@@ -452,6 +246,22 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
452
246
  """
453
247
  ...
454
248
 
249
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
250
+ """
251
+ Specifies that this step should execute on DGX cloud.
252
+
253
+
254
+ Parameters
255
+ ----------
256
+ gpu : int
257
+ Number of GPUs to use.
258
+ gpu_type : str
259
+ Type of Nvidia GPU to use.
260
+ queue_timeout : int
261
+ Time to keep the job in NVCF's queue.
262
+ """
263
+ ...
264
+
455
265
  @typing.overload
456
266
  def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
457
267
  """
@@ -509,42 +319,7 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
509
319
  """
510
320
  ...
511
321
 
512
- @typing.overload
513
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
514
- """
515
- Specifies secrets to be retrieved and injected as environment variables prior to
516
- the execution of a step.
517
-
518
-
519
- Parameters
520
- ----------
521
- sources : List[Union[str, Dict[str, Any]]], default: []
522
- List of secret specs, defining how the secrets are to be retrieved
523
- """
524
- ...
525
-
526
- @typing.overload
527
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
528
- ...
529
-
530
- @typing.overload
531
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
532
- ...
533
-
534
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
535
- """
536
- Specifies secrets to be retrieved and injected as environment variables prior to
537
- the execution of a step.
538
-
539
-
540
- Parameters
541
- ----------
542
- sources : List[Union[str, Dict[str, Any]]], default: []
543
- List of secret specs, defining how the secrets are to be retrieved
544
- """
545
- ...
546
-
547
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
322
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
548
323
  """
549
324
  Specifies that this step should execute on DGX cloud.
550
325
 
@@ -555,8 +330,6 @@ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[t
555
330
  Number of GPUs to use.
556
331
  gpu_type : str
557
332
  Type of Nvidia GPU to use.
558
- queue_timeout : int
559
- Time to keep the job in NVCF's queue.
560
333
  """
561
334
  ...
562
335
 
@@ -586,81 +359,150 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
586
359
  ...
587
360
 
588
361
  @typing.overload
589
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
362
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
590
363
  """
591
- Specifies the resources needed when executing this step.
364
+ Specifies a timeout for your step.
592
365
 
593
- Use `@resources` to specify the resource requirements
594
- independently of the specific compute layer (`@batch`, `@kubernetes`).
366
+ This decorator is useful if this step may hang indefinitely.
595
367
 
596
- You can choose the compute layer on the command line by executing e.g.
597
- ```
598
- python myflow.py run --with batch
599
- ```
600
- or
601
- ```
602
- python myflow.py run --with kubernetes
603
- ```
604
- which executes the flow on the desired system using the
605
- requirements specified in `@resources`.
368
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
369
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
370
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
371
+
372
+ Note that all the values specified in parameters are added together so if you specify
373
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
606
374
 
607
375
 
608
376
  Parameters
609
377
  ----------
610
- cpu : int, default 1
611
- Number of CPUs required for this step.
612
- gpu : int, optional, default None
613
- Number of GPUs required for this step.
614
- disk : int, optional, default None
615
- Disk size (in MB) required for this step. Only applies on Kubernetes.
616
- memory : int, default 4096
617
- Memory size (in MB) required for this step.
618
- shared_memory : int, optional, default None
619
- The value for the size (in MiB) of the /dev/shm volume for this step.
620
- This parameter maps to the `--shm-size` option in Docker.
378
+ seconds : int, default 0
379
+ Number of seconds to wait prior to timing out.
380
+ minutes : int, default 0
381
+ Number of minutes to wait prior to timing out.
382
+ hours : int, default 0
383
+ Number of hours to wait prior to timing out.
621
384
  """
622
385
  ...
623
386
 
624
387
  @typing.overload
625
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
388
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
626
389
  ...
627
390
 
628
391
  @typing.overload
629
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
392
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
630
393
  ...
631
394
 
632
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
395
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
633
396
  """
634
- Specifies the resources needed when executing this step.
397
+ Specifies a timeout for your step.
635
398
 
636
- Use `@resources` to specify the resource requirements
637
- independently of the specific compute layer (`@batch`, `@kubernetes`).
399
+ This decorator is useful if this step may hang indefinitely.
638
400
 
639
- You can choose the compute layer on the command line by executing e.g.
640
- ```
641
- python myflow.py run --with batch
642
- ```
643
- or
644
- ```
645
- python myflow.py run --with kubernetes
646
- ```
647
- which executes the flow on the desired system using the
648
- requirements specified in `@resources`.
401
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
402
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
403
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
404
+
405
+ Note that all the values specified in parameters are added together so if you specify
406
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
407
+
408
+
409
+ Parameters
410
+ ----------
411
+ seconds : int, default 0
412
+ Number of seconds to wait prior to timing out.
413
+ minutes : int, default 0
414
+ Number of minutes to wait prior to timing out.
415
+ hours : int, default 0
416
+ Number of hours to wait prior to timing out.
417
+ """
418
+ ...
419
+
420
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
421
+ """
422
+ Specifies that this step should execute on Kubernetes.
649
423
 
650
424
 
651
425
  Parameters
652
426
  ----------
653
427
  cpu : int, default 1
654
- Number of CPUs required for this step.
655
- gpu : int, optional, default None
656
- Number of GPUs required for this step.
657
- disk : int, optional, default None
658
- Disk size (in MB) required for this step. Only applies on Kubernetes.
428
+ Number of CPUs required for this step. If `@resources` is
429
+ also present, the maximum value from all decorators is used.
659
430
  memory : int, default 4096
660
- Memory size (in MB) required for this step.
661
- shared_memory : int, optional, default None
662
- The value for the size (in MiB) of the /dev/shm volume for this step.
663
- This parameter maps to the `--shm-size` option in Docker.
431
+ Memory size (in MB) required for this step. If
432
+ `@resources` is also present, the maximum value from all decorators is
433
+ used.
434
+ disk : int, default 10240
435
+ Disk size (in MB) required for this step. If
436
+ `@resources` is also present, the maximum value from all decorators is
437
+ used.
438
+ image : str, optional, default None
439
+ Docker image to use when launching on Kubernetes. If not specified, and
440
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
441
+ not, a default Docker image mapping to the current version of Python is used.
442
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
443
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
444
+ image_pull_secrets: List[str], default []
445
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
446
+ Kubernetes image pull secrets to use when pulling container images
447
+ in Kubernetes.
448
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
449
+ Kubernetes service account to use when launching pod in Kubernetes.
450
+ secrets : List[str], optional, default None
451
+ Kubernetes secrets to use when launching pod in Kubernetes. These
452
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
453
+ in Metaflow configuration.
454
+ node_selector: Union[Dict[str,str], str], optional, default None
455
+ Kubernetes node selector(s) to apply to the pod running the task.
456
+ Can be passed in as a comma separated string of values e.g.
457
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
458
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
459
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
460
+ Kubernetes namespace to use when launching pod in Kubernetes.
461
+ gpu : int, optional, default None
462
+ Number of GPUs required for this step. A value of zero implies that
463
+ the scheduled node should not have GPUs.
464
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
465
+ The vendor of the GPUs to be used for this step.
466
+ tolerations : List[str], default []
467
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
468
+ Kubernetes tolerations to use when launching pod in Kubernetes.
469
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
470
+ Kubernetes labels to use when launching pod in Kubernetes.
471
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
472
+ Kubernetes annotations to use when launching pod in Kubernetes.
473
+ use_tmpfs : bool, default False
474
+ This enables an explicit tmpfs mount for this step.
475
+ tmpfs_tempdir : bool, default True
476
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
477
+ tmpfs_size : int, optional, default: None
478
+ The value for the size (in MiB) of the tmpfs mount for this step.
479
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
480
+ memory allocated for this step.
481
+ tmpfs_path : str, optional, default /metaflow_temp
482
+ Path to tmpfs mount for this step.
483
+ persistent_volume_claims : Dict[str, str], optional, default None
484
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
485
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
486
+ shared_memory: int, optional
487
+ Shared memory size (in MiB) required for this step
488
+ port: int, optional
489
+ Port number to specify in the Kubernetes job object
490
+ compute_pool : str, optional, default None
491
+ Compute pool to be used for for this step.
492
+ If not specified, any accessible compute pool within the perimeter is used.
493
+ hostname_resolution_timeout: int, default 10 * 60
494
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
495
+ Only applicable when @parallel is used.
496
+ qos: str, default: Burstable
497
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
498
+
499
+ security_context: Dict[str, Any], optional, default None
500
+ Container security context. Applies to the task container. Allows the following keys:
501
+ - privileged: bool, optional, default None
502
+ - allow_privilege_escalation: bool, optional, default None
503
+ - run_as_user: int, optional, default None
504
+ - run_as_group: int, optional, default None
505
+ - run_as_non_root: bool, optional, default None
664
506
  """
665
507
  ...
666
508
 
@@ -763,139 +605,167 @@ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy:
763
605
  ...
764
606
 
765
607
  @typing.overload
766
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
608
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
767
609
  """
768
- Specifies a timeout for your step.
769
-
770
- This decorator is useful if this step may hang indefinitely.
610
+ Specifies the resources needed when executing this step.
771
611
 
772
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
773
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
774
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
612
+ Use `@resources` to specify the resource requirements
613
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
775
614
 
776
- Note that all the values specified in parameters are added together so if you specify
777
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
615
+ You can choose the compute layer on the command line by executing e.g.
616
+ ```
617
+ python myflow.py run --with batch
618
+ ```
619
+ or
620
+ ```
621
+ python myflow.py run --with kubernetes
622
+ ```
623
+ which executes the flow on the desired system using the
624
+ requirements specified in `@resources`.
778
625
 
779
626
 
780
627
  Parameters
781
628
  ----------
782
- seconds : int, default 0
783
- Number of seconds to wait prior to timing out.
784
- minutes : int, default 0
785
- Number of minutes to wait prior to timing out.
786
- hours : int, default 0
787
- Number of hours to wait prior to timing out.
629
+ cpu : int, default 1
630
+ Number of CPUs required for this step.
631
+ gpu : int, optional, default None
632
+ Number of GPUs required for this step.
633
+ disk : int, optional, default None
634
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
635
+ memory : int, default 4096
636
+ Memory size (in MB) required for this step.
637
+ shared_memory : int, optional, default None
638
+ The value for the size (in MiB) of the /dev/shm volume for this step.
639
+ This parameter maps to the `--shm-size` option in Docker.
788
640
  """
789
641
  ...
790
642
 
791
643
  @typing.overload
792
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
644
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
793
645
  ...
794
646
 
795
647
  @typing.overload
796
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
648
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
797
649
  ...
798
650
 
799
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
651
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
800
652
  """
801
- Specifies a timeout for your step.
653
+ Specifies the resources needed when executing this step.
802
654
 
803
- This decorator is useful if this step may hang indefinitely.
804
-
805
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
806
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
807
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
655
+ Use `@resources` to specify the resource requirements
656
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
808
657
 
809
- Note that all the values specified in parameters are added together so if you specify
810
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
658
+ You can choose the compute layer on the command line by executing e.g.
659
+ ```
660
+ python myflow.py run --with batch
661
+ ```
662
+ or
663
+ ```
664
+ python myflow.py run --with kubernetes
665
+ ```
666
+ which executes the flow on the desired system using the
667
+ requirements specified in `@resources`.
811
668
 
812
669
 
813
670
  Parameters
814
671
  ----------
815
- seconds : int, default 0
816
- Number of seconds to wait prior to timing out.
817
- minutes : int, default 0
818
- Number of minutes to wait prior to timing out.
819
- hours : int, default 0
820
- Number of hours to wait prior to timing out.
672
+ cpu : int, default 1
673
+ Number of CPUs required for this step.
674
+ gpu : int, optional, default None
675
+ Number of GPUs required for this step.
676
+ disk : int, optional, default None
677
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
678
+ memory : int, default 4096
679
+ Memory size (in MB) required for this step.
680
+ shared_memory : int, optional, default None
681
+ The value for the size (in MiB) of the /dev/shm volume for this step.
682
+ This parameter maps to the `--shm-size` option in Docker.
821
683
  """
822
684
  ...
823
685
 
824
686
  @typing.overload
825
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
687
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
826
688
  """
827
- Specifies the Conda environment for the step.
689
+ Specifies that the step will success under all circumstances.
828
690
 
829
- Information in this decorator will augment any
830
- attributes set in the `@conda_base` flow-level decorator. Hence,
831
- you can use `@conda_base` to set packages required by all
832
- steps and use `@conda` to specify step-specific overrides.
691
+ The decorator will create an optional artifact, specified by `var`, which
692
+ contains the exception raised. You can use it to detect the presence
693
+ of errors, indicating that all happy-path artifacts produced by the step
694
+ are missing.
833
695
 
834
696
 
835
697
  Parameters
836
698
  ----------
837
- packages : Dict[str, str], default {}
838
- Packages to use for this step. The key is the name of the package
839
- and the value is the version to use.
840
- libraries : Dict[str, str], default {}
841
- Supported for backward compatibility. When used with packages, packages will take precedence.
842
- python : str, optional, default None
843
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
844
- that the version used will correspond to the version of the Python interpreter used to start the run.
845
- disabled : bool, default False
846
- If set to True, disables @conda.
699
+ var : str, optional, default None
700
+ Name of the artifact in which to store the caught exception.
701
+ If not specified, the exception is not stored.
702
+ print_exception : bool, default True
703
+ Determines whether or not the exception is printed to
704
+ stdout when caught.
847
705
  """
848
706
  ...
849
707
 
850
708
  @typing.overload
851
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
709
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
852
710
  ...
853
711
 
854
712
  @typing.overload
855
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
713
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
856
714
  ...
857
715
 
858
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
716
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
859
717
  """
860
- Specifies the Conda environment for the step.
718
+ Specifies that the step will success under all circumstances.
861
719
 
862
- Information in this decorator will augment any
863
- attributes set in the `@conda_base` flow-level decorator. Hence,
864
- you can use `@conda_base` to set packages required by all
865
- steps and use `@conda` to specify step-specific overrides.
720
+ The decorator will create an optional artifact, specified by `var`, which
721
+ contains the exception raised. You can use it to detect the presence
722
+ of errors, indicating that all happy-path artifacts produced by the step
723
+ are missing.
866
724
 
867
725
 
868
726
  Parameters
869
727
  ----------
870
- packages : Dict[str, str], default {}
871
- Packages to use for this step. The key is the name of the package
872
- and the value is the version to use.
873
- libraries : Dict[str, str], default {}
874
- Supported for backward compatibility. When used with packages, packages will take precedence.
875
- python : str, optional, default None
876
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
877
- that the version used will correspond to the version of the Python interpreter used to start the run.
878
- disabled : bool, default False
879
- If set to True, disables @conda.
728
+ var : str, optional, default None
729
+ Name of the artifact in which to store the caught exception.
730
+ If not specified, the exception is not stored.
731
+ print_exception : bool, default True
732
+ Determines whether or not the exception is printed to
733
+ stdout when caught.
880
734
  """
881
735
  ...
882
736
 
883
737
  @typing.overload
884
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
738
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
885
739
  """
886
- Decorator prototype for all step decorators. This function gets specialized
887
- and imported for all decorators types by _import_plugin_decorators().
740
+ Specifies secrets to be retrieved and injected as environment variables prior to
741
+ the execution of a step.
742
+
743
+
744
+ Parameters
745
+ ----------
746
+ sources : List[Union[str, Dict[str, Any]]], default: []
747
+ List of secret specs, defining how the secrets are to be retrieved
888
748
  """
889
749
  ...
890
750
 
891
751
  @typing.overload
892
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
752
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
893
753
  ...
894
754
 
895
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
755
+ @typing.overload
756
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
757
+ ...
758
+
759
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
896
760
  """
897
- Decorator prototype for all step decorators. This function gets specialized
898
- and imported for all decorators types by _import_plugin_decorators().
761
+ Specifies secrets to be retrieved and injected as environment variables prior to
762
+ the execution of a step.
763
+
764
+
765
+ Parameters
766
+ ----------
767
+ sources : List[Union[str, Dict[str, Any]]], default: []
768
+ List of secret specs, defining how the secrets are to be retrieved
899
769
  """
900
770
  ...
901
771
 
@@ -932,32 +802,69 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
932
802
  """
933
803
  ...
934
804
 
935
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
805
+ @typing.overload
806
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
936
807
  """
937
- Specifies that this step is used to deploy an instance of the app.
938
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
808
+ Creates a human-readable report, a Metaflow Card, after this step completes.
809
+
810
+ Note that you may add multiple `@card` decorators in a step with different parameters.
939
811
 
940
812
 
941
813
  Parameters
942
814
  ----------
943
- app_port : int
944
- Number of GPUs to use.
945
- app_name : str
946
- Name of the app to deploy.
815
+ type : str, default 'default'
816
+ Card type.
817
+ id : str, optional, default None
818
+ If multiple cards are present, use this id to identify this card.
819
+ options : Dict[str, Any], default {}
820
+ Options passed to the card. The contents depend on the card type.
821
+ timeout : int, default 45
822
+ Interrupt reporting if it takes more than this many seconds.
947
823
  """
948
824
  ...
949
825
 
950
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
826
+ @typing.overload
827
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
828
+ ...
829
+
830
+ @typing.overload
831
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
832
+ ...
833
+
834
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
951
835
  """
952
- Specifies that this step should execute on DGX cloud.
836
+ Creates a human-readable report, a Metaflow Card, after this step completes.
837
+
838
+ Note that you may add multiple `@card` decorators in a step with different parameters.
953
839
 
954
840
 
955
841
  Parameters
956
842
  ----------
957
- gpu : int
958
- Number of GPUs to use.
959
- gpu_type : str
960
- Type of Nvidia GPU to use.
843
+ type : str, default 'default'
844
+ Card type.
845
+ id : str, optional, default None
846
+ If multiple cards are present, use this id to identify this card.
847
+ options : Dict[str, Any], default {}
848
+ Options passed to the card. The contents depend on the card type.
849
+ timeout : int, default 45
850
+ Interrupt reporting if it takes more than this many seconds.
851
+ """
852
+ ...
853
+
854
+ @typing.overload
855
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
856
+ """
857
+ Internal decorator to support Fast bakery
858
+ """
859
+ ...
860
+
861
+ @typing.overload
862
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
863
+ ...
864
+
865
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
866
+ """
867
+ Internal decorator to support Fast bakery
961
868
  """
962
869
  ...
963
870
 
@@ -1000,19 +907,325 @@ def vllm(*, model: str, backend: str, debug: bool, kwargs: typing.Any) -> typing
1000
907
  ...
1001
908
 
1002
909
  @typing.overload
1003
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
910
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1004
911
  """
1005
- Internal decorator to support Fast bakery
912
+ Specifies the Conda environment for the step.
913
+
914
+ Information in this decorator will augment any
915
+ attributes set in the `@conda_base` flow-level decorator. Hence,
916
+ you can use `@conda_base` to set packages required by all
917
+ steps and use `@conda` to specify step-specific overrides.
918
+
919
+
920
+ Parameters
921
+ ----------
922
+ packages : Dict[str, str], default {}
923
+ Packages to use for this step. The key is the name of the package
924
+ and the value is the version to use.
925
+ libraries : Dict[str, str], default {}
926
+ Supported for backward compatibility. When used with packages, packages will take precedence.
927
+ python : str, optional, default None
928
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
929
+ that the version used will correspond to the version of the Python interpreter used to start the run.
930
+ disabled : bool, default False
931
+ If set to True, disables @conda.
1006
932
  """
1007
933
  ...
1008
934
 
1009
935
  @typing.overload
1010
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
936
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
937
+ ...
938
+
939
+ @typing.overload
940
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
941
+ ...
942
+
943
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
944
+ """
945
+ Specifies the Conda environment for the step.
946
+
947
+ Information in this decorator will augment any
948
+ attributes set in the `@conda_base` flow-level decorator. Hence,
949
+ you can use `@conda_base` to set packages required by all
950
+ steps and use `@conda` to specify step-specific overrides.
951
+
952
+
953
+ Parameters
954
+ ----------
955
+ packages : Dict[str, str], default {}
956
+ Packages to use for this step. The key is the name of the package
957
+ and the value is the version to use.
958
+ libraries : Dict[str, str], default {}
959
+ Supported for backward compatibility. When used with packages, packages will take precedence.
960
+ python : str, optional, default None
961
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
962
+ that the version used will correspond to the version of the Python interpreter used to start the run.
963
+ disabled : bool, default False
964
+ If set to True, disables @conda.
965
+ """
966
+ ...
967
+
968
+ @typing.overload
969
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
970
+ """
971
+ Specifies the PyPI packages for the step.
972
+
973
+ Information in this decorator will augment any
974
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
975
+ you can use `@pypi_base` to set packages required by all
976
+ steps and use `@pypi` to specify step-specific overrides.
977
+
978
+
979
+ Parameters
980
+ ----------
981
+ packages : Dict[str, str], default: {}
982
+ Packages to use for this step. The key is the name of the package
983
+ and the value is the version to use.
984
+ python : str, optional, default: None
985
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
986
+ that the version used will correspond to the version of the Python interpreter used to start the run.
987
+ """
988
+ ...
989
+
990
+ @typing.overload
991
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
992
+ ...
993
+
994
+ @typing.overload
995
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
996
+ ...
997
+
998
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
999
+ """
1000
+ Specifies the PyPI packages for the step.
1001
+
1002
+ Information in this decorator will augment any
1003
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1004
+ you can use `@pypi_base` to set packages required by all
1005
+ steps and use `@pypi` to specify step-specific overrides.
1006
+
1007
+
1008
+ Parameters
1009
+ ----------
1010
+ packages : Dict[str, str], default: {}
1011
+ Packages to use for this step. The key is the name of the package
1012
+ and the value is the version to use.
1013
+ python : str, optional, default: None
1014
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1015
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1016
+ """
1017
+ ...
1018
+
1019
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1020
+ """
1021
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1022
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1023
+
1024
+
1025
+ Parameters
1026
+ ----------
1027
+ timeout : int
1028
+ Time, in seconds before the task times out and fails. (Default: 3600)
1029
+ poke_interval : int
1030
+ Time in seconds that the job should wait in between each try. (Default: 60)
1031
+ mode : str
1032
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1033
+ exponential_backoff : bool
1034
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1035
+ pool : str
1036
+ the slot pool this task should run in,
1037
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1038
+ soft_fail : bool
1039
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1040
+ name : str
1041
+ Name of the sensor on Airflow
1042
+ description : str
1043
+ Description of sensor in the Airflow UI
1044
+ external_dag_id : str
1045
+ The dag_id that contains the task you want to wait for.
1046
+ external_task_ids : List[str]
1047
+ The list of task_ids that you want to wait for.
1048
+ If None (default value) the sensor waits for the DAG. (Default: None)
1049
+ allowed_states : List[str]
1050
+ Iterable of allowed states, (Default: ['success'])
1051
+ failed_states : List[str]
1052
+ Iterable of failed or dis-allowed states. (Default: None)
1053
+ execution_delta : datetime.timedelta
1054
+ time difference with the previous execution to look at,
1055
+ the default is the same logical date as the current task or DAG. (Default: None)
1056
+ check_existence: bool
1057
+ Set to True to check if the external task exists or check if
1058
+ the DAG to wait for exists. (Default: True)
1059
+ """
1060
+ ...
1061
+
1062
+ @typing.overload
1063
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1064
+ """
1065
+ Specifies the PyPI packages for all steps of the flow.
1066
+
1067
+ Use `@pypi_base` to set common packages required by all
1068
+ steps and use `@pypi` to specify step-specific overrides.
1069
+
1070
+ Parameters
1071
+ ----------
1072
+ packages : Dict[str, str], default: {}
1073
+ Packages to use for this flow. The key is the name of the package
1074
+ and the value is the version to use.
1075
+ python : str, optional, default: None
1076
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1077
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1078
+ """
1079
+ ...
1080
+
1081
+ @typing.overload
1082
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1083
+ ...
1084
+
1085
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1086
+ """
1087
+ Specifies the PyPI packages for all steps of the flow.
1088
+
1089
+ Use `@pypi_base` to set common packages required by all
1090
+ steps and use `@pypi` to specify step-specific overrides.
1091
+
1092
+ Parameters
1093
+ ----------
1094
+ packages : Dict[str, str], default: {}
1095
+ Packages to use for this flow. The key is the name of the package
1096
+ and the value is the version to use.
1097
+ python : str, optional, default: None
1098
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1099
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1100
+ """
1101
+ ...
1102
+
1103
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1104
+ """
1105
+ Specifies what flows belong to the same project.
1106
+
1107
+ A project-specific namespace is created for all flows that
1108
+ use the same `@project(name)`.
1109
+
1110
+
1111
+ Parameters
1112
+ ----------
1113
+ name : str
1114
+ Project name. Make sure that the name is unique amongst all
1115
+ projects that use the same production scheduler. The name may
1116
+ contain only lowercase alphanumeric characters and underscores.
1117
+
1118
+ branch : Optional[str], default None
1119
+ The branch to use. If not specified, the branch is set to
1120
+ `user.<username>` unless `production` is set to `True`. This can
1121
+ also be set on the command line using `--branch` as a top-level option.
1122
+ It is an error to specify `branch` in the decorator and on the command line.
1123
+
1124
+ production : bool, default False
1125
+ Whether or not the branch is the production branch. This can also be set on the
1126
+ command line using `--production` as a top-level option. It is an error to specify
1127
+ `production` in the decorator and on the command line.
1128
+ The project branch name will be:
1129
+ - if `branch` is specified:
1130
+ - if `production` is True: `prod.<branch>`
1131
+ - if `production` is False: `test.<branch>`
1132
+ - if `branch` is not specified:
1133
+ - if `production` is True: `prod`
1134
+ - if `production` is False: `user.<username>`
1135
+ """
1136
+ ...
1137
+
1138
+ @typing.overload
1139
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1140
+ """
1141
+ Specifies the times when the flow should be run when running on a
1142
+ production scheduler.
1143
+
1144
+
1145
+ Parameters
1146
+ ----------
1147
+ hourly : bool, default False
1148
+ Run the workflow hourly.
1149
+ daily : bool, default True
1150
+ Run the workflow daily.
1151
+ weekly : bool, default False
1152
+ Run the workflow weekly.
1153
+ cron : str, optional, default None
1154
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1155
+ specified by this expression.
1156
+ timezone : str, optional, default None
1157
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1158
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1159
+ """
1160
+ ...
1161
+
1162
+ @typing.overload
1163
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1164
+ ...
1165
+
1166
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1167
+ """
1168
+ Specifies the times when the flow should be run when running on a
1169
+ production scheduler.
1170
+
1171
+
1172
+ Parameters
1173
+ ----------
1174
+ hourly : bool, default False
1175
+ Run the workflow hourly.
1176
+ daily : bool, default True
1177
+ Run the workflow daily.
1178
+ weekly : bool, default False
1179
+ Run the workflow weekly.
1180
+ cron : str, optional, default None
1181
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1182
+ specified by this expression.
1183
+ timezone : str, optional, default None
1184
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1185
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1186
+ """
1011
1187
  ...
1012
1188
 
1013
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1189
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1014
1190
  """
1015
- Internal decorator to support Fast bakery
1191
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1192
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1193
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1194
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1195
+ starts only after all sensors finish.
1196
+
1197
+
1198
+ Parameters
1199
+ ----------
1200
+ timeout : int
1201
+ Time, in seconds before the task times out and fails. (Default: 3600)
1202
+ poke_interval : int
1203
+ Time in seconds that the job should wait in between each try. (Default: 60)
1204
+ mode : str
1205
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1206
+ exponential_backoff : bool
1207
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1208
+ pool : str
1209
+ the slot pool this task should run in,
1210
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1211
+ soft_fail : bool
1212
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1213
+ name : str
1214
+ Name of the sensor on Airflow
1215
+ description : str
1216
+ Description of sensor in the Airflow UI
1217
+ bucket_key : Union[str, List[str]]
1218
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1219
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1220
+ bucket_name : str
1221
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1222
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1223
+ wildcard_match : bool
1224
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1225
+ aws_conn_id : str
1226
+ a reference to the s3 connection on Airflow. (Default: None)
1227
+ verify : bool
1228
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1016
1229
  """
1017
1230
  ...
1018
1231
 
@@ -1130,125 +1343,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1130
1343
  """
1131
1344
  ...
1132
1345
 
1133
- @typing.overload
1134
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1135
- """
1136
- Specifies the PyPI packages for all steps of the flow.
1137
-
1138
- Use `@pypi_base` to set common packages required by all
1139
- steps and use `@pypi` to specify step-specific overrides.
1140
-
1141
- Parameters
1142
- ----------
1143
- packages : Dict[str, str], default: {}
1144
- Packages to use for this flow. The key is the name of the package
1145
- and the value is the version to use.
1146
- python : str, optional, default: None
1147
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1148
- that the version used will correspond to the version of the Python interpreter used to start the run.
1149
- """
1150
- ...
1151
-
1152
- @typing.overload
1153
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1154
- ...
1155
-
1156
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1157
- """
1158
- Specifies the PyPI packages for all steps of the flow.
1159
-
1160
- Use `@pypi_base` to set common packages required by all
1161
- steps and use `@pypi` to specify step-specific overrides.
1162
-
1163
- Parameters
1164
- ----------
1165
- packages : Dict[str, str], default: {}
1166
- Packages to use for this flow. The key is the name of the package
1167
- and the value is the version to use.
1168
- python : str, optional, default: None
1169
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1170
- that the version used will correspond to the version of the Python interpreter used to start the run.
1171
- """
1172
- ...
1173
-
1174
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1175
- """
1176
- Specifies what flows belong to the same project.
1177
-
1178
- A project-specific namespace is created for all flows that
1179
- use the same `@project(name)`.
1180
-
1181
-
1182
- Parameters
1183
- ----------
1184
- name : str
1185
- Project name. Make sure that the name is unique amongst all
1186
- projects that use the same production scheduler. The name may
1187
- contain only lowercase alphanumeric characters and underscores.
1188
-
1189
- branch : Optional[str], default None
1190
- The branch to use. If not specified, the branch is set to
1191
- `user.<username>` unless `production` is set to `True`. This can
1192
- also be set on the command line using `--branch` as a top-level option.
1193
- It is an error to specify `branch` in the decorator and on the command line.
1194
-
1195
- production : bool, default False
1196
- Whether or not the branch is the production branch. This can also be set on the
1197
- command line using `--production` as a top-level option. It is an error to specify
1198
- `production` in the decorator and on the command line.
1199
- The project branch name will be:
1200
- - if `branch` is specified:
1201
- - if `production` is True: `prod.<branch>`
1202
- - if `production` is False: `test.<branch>`
1203
- - if `branch` is not specified:
1204
- - if `production` is True: `prod`
1205
- - if `production` is False: `user.<username>`
1206
- """
1207
- ...
1208
-
1209
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1210
- """
1211
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1212
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1213
-
1214
-
1215
- Parameters
1216
- ----------
1217
- timeout : int
1218
- Time, in seconds before the task times out and fails. (Default: 3600)
1219
- poke_interval : int
1220
- Time in seconds that the job should wait in between each try. (Default: 60)
1221
- mode : str
1222
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1223
- exponential_backoff : bool
1224
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1225
- pool : str
1226
- the slot pool this task should run in,
1227
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1228
- soft_fail : bool
1229
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1230
- name : str
1231
- Name of the sensor on Airflow
1232
- description : str
1233
- Description of sensor in the Airflow UI
1234
- external_dag_id : str
1235
- The dag_id that contains the task you want to wait for.
1236
- external_task_ids : List[str]
1237
- The list of task_ids that you want to wait for.
1238
- If None (default value) the sensor waits for the DAG. (Default: None)
1239
- allowed_states : List[str]
1240
- Iterable of allowed states, (Default: ['success'])
1241
- failed_states : List[str]
1242
- Iterable of failed or dis-allowed states. (Default: None)
1243
- execution_delta : datetime.timedelta
1244
- time difference with the previous execution to look at,
1245
- the default is the same logical date as the current task or DAG. (Default: None)
1246
- check_existence: bool
1247
- Set to True to check if the external task exists or check if
1248
- the DAG to wait for exists. (Default: True)
1249
- """
1250
- ...
1251
-
1252
1346
  @typing.overload
1253
1347
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1254
1348
  """
@@ -1494,99 +1588,5 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1494
1588
  """
1495
1589
  ...
1496
1590
 
1497
- @typing.overload
1498
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1499
- """
1500
- Specifies the times when the flow should be run when running on a
1501
- production scheduler.
1502
-
1503
-
1504
- Parameters
1505
- ----------
1506
- hourly : bool, default False
1507
- Run the workflow hourly.
1508
- daily : bool, default True
1509
- Run the workflow daily.
1510
- weekly : bool, default False
1511
- Run the workflow weekly.
1512
- cron : str, optional, default None
1513
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1514
- specified by this expression.
1515
- timezone : str, optional, default None
1516
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1517
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1518
- """
1519
- ...
1520
-
1521
- @typing.overload
1522
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1523
- ...
1524
-
1525
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1526
- """
1527
- Specifies the times when the flow should be run when running on a
1528
- production scheduler.
1529
-
1530
-
1531
- Parameters
1532
- ----------
1533
- hourly : bool, default False
1534
- Run the workflow hourly.
1535
- daily : bool, default True
1536
- Run the workflow daily.
1537
- weekly : bool, default False
1538
- Run the workflow weekly.
1539
- cron : str, optional, default None
1540
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1541
- specified by this expression.
1542
- timezone : str, optional, default None
1543
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1544
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1545
- """
1546
- ...
1547
-
1548
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1549
- """
1550
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1551
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1552
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1553
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1554
- starts only after all sensors finish.
1555
-
1556
-
1557
- Parameters
1558
- ----------
1559
- timeout : int
1560
- Time, in seconds before the task times out and fails. (Default: 3600)
1561
- poke_interval : int
1562
- Time in seconds that the job should wait in between each try. (Default: 60)
1563
- mode : str
1564
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1565
- exponential_backoff : bool
1566
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1567
- pool : str
1568
- the slot pool this task should run in,
1569
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1570
- soft_fail : bool
1571
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1572
- name : str
1573
- Name of the sensor on Airflow
1574
- description : str
1575
- Description of sensor in the Airflow UI
1576
- bucket_key : Union[str, List[str]]
1577
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1578
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1579
- bucket_name : str
1580
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1581
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1582
- wildcard_match : bool
1583
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1584
- aws_conn_id : str
1585
- a reference to the s3 connection on Airflow. (Default: None)
1586
- verify : bool
1587
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1588
- """
1589
- ...
1590
-
1591
1591
  pkg_name: str
1592
1592