ob-metaflow-stubs 6.0.3.179rc3__py2.py3-none-any.whl → 6.0.3.179rc5__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. metaflow-stubs/__init__.pyi +608 -608
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +4 -4
  8. metaflow-stubs/client/filecache.pyi +1 -1
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +3 -3
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +2 -2
  14. metaflow-stubs/info_file.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +1 -1
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +53 -53
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +1 -1
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +4 -4
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +3 -3
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +1 -1
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +1 -1
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +1 -1
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +1 -1
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +1 -1
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +1 -1
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +2 -2
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +1 -1
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  87. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  88. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  89. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
  90. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  91. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
  92. metaflow-stubs/multicore_utils.pyi +1 -1
  93. metaflow-stubs/ob_internal.pyi +1 -1
  94. metaflow-stubs/parameters.pyi +2 -2
  95. metaflow-stubs/plugins/__init__.pyi +9 -9
  96. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  97. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  98. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  99. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  100. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  101. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  102. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  103. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  104. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  105. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  106. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  107. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  108. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  109. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  110. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  111. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  112. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  113. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  114. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  115. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  116. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  117. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  118. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  119. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  120. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  121. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  122. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  123. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  124. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +1 -1
  125. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  126. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  127. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  128. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  129. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  130. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  131. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  132. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  133. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  134. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  135. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  136. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  137. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  138. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  139. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  140. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  141. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  142. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  143. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  144. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  145. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  146. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  147. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  148. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  149. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  150. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  151. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  152. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  153. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  154. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  155. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  156. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  157. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  158. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  159. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  160. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  161. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  162. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  163. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  164. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  165. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  166. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  167. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  168. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
  169. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  170. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  171. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  172. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  173. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  174. metaflow-stubs/plugins/ollama/__init__.pyi +1 -1
  175. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  176. metaflow-stubs/plugins/perimeters.pyi +1 -1
  177. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  178. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  179. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  180. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  181. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  182. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  183. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  184. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  185. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  186. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  187. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  188. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  189. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  190. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  191. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  192. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
  193. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  194. metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
  195. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  196. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  197. metaflow-stubs/profilers/__init__.pyi +1 -1
  198. metaflow-stubs/pylint_wrapper.pyi +1 -1
  199. metaflow-stubs/runner/__init__.pyi +1 -1
  200. metaflow-stubs/runner/deployer.pyi +3 -3
  201. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  202. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  203. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  204. metaflow-stubs/runner/nbrun.pyi +1 -1
  205. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  206. metaflow-stubs/runner/utils.pyi +2 -2
  207. metaflow-stubs/system/__init__.pyi +1 -1
  208. metaflow-stubs/system/system_logger.pyi +1 -1
  209. metaflow-stubs/system/system_monitor.pyi +1 -1
  210. metaflow-stubs/tagging_util.pyi +1 -1
  211. metaflow-stubs/tuple_util.pyi +1 -1
  212. metaflow-stubs/user_configs/__init__.pyi +1 -1
  213. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  214. metaflow-stubs/user_configs/config_options.pyi +1 -1
  215. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  216. {ob_metaflow_stubs-6.0.3.179rc3.dist-info → ob_metaflow_stubs-6.0.3.179rc5.dist-info}/METADATA +1 -1
  217. ob_metaflow_stubs-6.0.3.179rc5.dist-info/RECORD +220 -0
  218. ob_metaflow_stubs-6.0.3.179rc3.dist-info/RECORD +0 -220
  219. {ob_metaflow_stubs-6.0.3.179rc3.dist-info → ob_metaflow_stubs-6.0.3.179rc5.dist-info}/WHEEL +0 -0
  220. {ob_metaflow_stubs-6.0.3.179rc3.dist-info → ob_metaflow_stubs-6.0.3.179rc5.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.15.17.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-06-13T20:43:51.355806 #
4
+ # Generated on 2025-06-13T21:29:38.839071 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -37,16 +37,16 @@ from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDec
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
38
  from . import tuple_util as tuple_util
39
39
  from . import cards as cards
40
- from . import events as events
41
40
  from . import metaflow_git as metaflow_git
41
+ from . import events as events
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
47
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
48
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
49
48
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
49
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
52
52
  from .client.core import get_namespace as get_namespace
@@ -155,6 +155,39 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
155
155
  """
156
156
  ...
157
157
 
158
+ @typing.overload
159
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
160
+ """
161
+ Specifies environment variables to be set prior to the execution of a step.
162
+
163
+
164
+ Parameters
165
+ ----------
166
+ vars : Dict[str, str], default {}
167
+ Dictionary of environment variables to set.
168
+ """
169
+ ...
170
+
171
+ @typing.overload
172
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
173
+ ...
174
+
175
+ @typing.overload
176
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
177
+ ...
178
+
179
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
180
+ """
181
+ Specifies environment variables to be set prior to the execution of a step.
182
+
183
+
184
+ Parameters
185
+ ----------
186
+ vars : Dict[str, str], default {}
187
+ Dictionary of environment variables to set.
188
+ """
189
+ ...
190
+
158
191
  @typing.overload
159
192
  def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
160
193
  """
@@ -213,71 +246,53 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
213
246
  ...
214
247
 
215
248
  @typing.overload
216
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
249
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
217
250
  """
218
- Specifies the number of times the task corresponding
219
- to a step needs to be retried.
220
-
221
- This decorator is useful for handling transient errors, such as networking issues.
222
- If your task contains operations that can't be retried safely, e.g. database updates,
223
- it is advisable to annotate it with `@retry(times=0)`.
251
+ Specifies that the step will success under all circumstances.
224
252
 
225
- This can be used in conjunction with the `@catch` decorator. The `@catch`
226
- decorator will execute a no-op task after all retries have been exhausted,
227
- ensuring that the flow execution can continue.
253
+ The decorator will create an optional artifact, specified by `var`, which
254
+ contains the exception raised. You can use it to detect the presence
255
+ of errors, indicating that all happy-path artifacts produced by the step
256
+ are missing.
228
257
 
229
258
 
230
259
  Parameters
231
260
  ----------
232
- times : int, default 3
233
- Number of times to retry this task.
234
- minutes_between_retries : int, default 2
235
- Number of minutes between retries.
261
+ var : str, optional, default None
262
+ Name of the artifact in which to store the caught exception.
263
+ If not specified, the exception is not stored.
264
+ print_exception : bool, default True
265
+ Determines whether or not the exception is printed to
266
+ stdout when caught.
236
267
  """
237
268
  ...
238
269
 
239
270
  @typing.overload
240
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
271
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
241
272
  ...
242
273
 
243
274
  @typing.overload
244
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
275
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
245
276
  ...
246
277
 
247
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
278
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
248
279
  """
249
- Specifies the number of times the task corresponding
250
- to a step needs to be retried.
251
-
252
- This decorator is useful for handling transient errors, such as networking issues.
253
- If your task contains operations that can't be retried safely, e.g. database updates,
254
- it is advisable to annotate it with `@retry(times=0)`.
255
-
256
- This can be used in conjunction with the `@catch` decorator. The `@catch`
257
- decorator will execute a no-op task after all retries have been exhausted,
258
- ensuring that the flow execution can continue.
259
-
280
+ Specifies that the step will success under all circumstances.
260
281
 
261
- Parameters
262
- ----------
263
- times : int, default 3
264
- Number of times to retry this task.
265
- minutes_between_retries : int, default 2
266
- Number of minutes between retries.
267
- """
268
- ...
269
-
270
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
271
- """
272
- Specifies that this step should execute on DGX cloud.
282
+ The decorator will create an optional artifact, specified by `var`, which
283
+ contains the exception raised. You can use it to detect the presence
284
+ of errors, indicating that all happy-path artifacts produced by the step
285
+ are missing.
273
286
 
274
287
 
275
288
  Parameters
276
289
  ----------
277
- gpu : int
278
- Number of GPUs to use.
279
- gpu_type : str
280
- Type of Nvidia GPU to use.
290
+ var : str, optional, default None
291
+ Name of the artifact in which to store the caught exception.
292
+ If not specified, the exception is not stored.
293
+ print_exception : bool, default True
294
+ Determines whether or not the exception is printed to
295
+ stdout when caught.
281
296
  """
282
297
  ...
283
298
 
@@ -370,82 +385,6 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
370
385
  """
371
386
  ...
372
387
 
373
- def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
374
- """
375
- This decorator is used to run Ollama APIs as Metaflow task sidecars.
376
-
377
- User code call
378
- --------------
379
- @ollama(
380
- models=[...],
381
- ...
382
- )
383
-
384
- Valid backend options
385
- ---------------------
386
- - 'local': Run as a separate process on the local task machine.
387
- - (TODO) 'managed': Outerbounds hosts and selects compute provider.
388
- - (TODO) 'remote': Spin up separate instance to serve Ollama models.
389
-
390
- Valid model options
391
- -------------------
392
- Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
393
-
394
-
395
- Parameters
396
- ----------
397
- models: list[str]
398
- List of Ollama containers running models in sidecars.
399
- backend: str
400
- Determines where and how to run the Ollama process.
401
- force_pull: bool
402
- Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
403
- cache_update_policy: str
404
- Cache update policy: "auto", "force", or "never".
405
- force_cache_update: bool
406
- Simple override for "force" cache update policy.
407
- debug: bool
408
- Whether to turn on verbose debugging logs.
409
- circuit_breaker_config: dict
410
- Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
411
- timeout_config: dict
412
- Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
413
- """
414
- ...
415
-
416
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
417
- """
418
- Specifies that this step should execute on DGX cloud.
419
-
420
-
421
- Parameters
422
- ----------
423
- gpu : int
424
- Number of GPUs to use.
425
- gpu_type : str
426
- Type of Nvidia GPU to use.
427
- queue_timeout : int
428
- Time to keep the job in NVCF's queue.
429
- """
430
- ...
431
-
432
- @typing.overload
433
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
434
- """
435
- Internal decorator to support Fast bakery
436
- """
437
- ...
438
-
439
- @typing.overload
440
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
441
- ...
442
-
443
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
444
- """
445
- Internal decorator to support Fast bakery
446
- """
447
- ...
448
-
449
388
  @typing.overload
450
389
  def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
451
390
  """
@@ -525,6 +464,70 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
525
464
  """
526
465
  ...
527
466
 
467
+ @typing.overload
468
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
469
+ """
470
+ Creates a human-readable report, a Metaflow Card, after this step completes.
471
+
472
+ Note that you may add multiple `@card` decorators in a step with different parameters.
473
+
474
+
475
+ Parameters
476
+ ----------
477
+ type : str, default 'default'
478
+ Card type.
479
+ id : str, optional, default None
480
+ If multiple cards are present, use this id to identify this card.
481
+ options : Dict[str, Any], default {}
482
+ Options passed to the card. The contents depend on the card type.
483
+ timeout : int, default 45
484
+ Interrupt reporting if it takes more than this many seconds.
485
+ """
486
+ ...
487
+
488
+ @typing.overload
489
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
490
+ ...
491
+
492
+ @typing.overload
493
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
494
+ ...
495
+
496
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
497
+ """
498
+ Creates a human-readable report, a Metaflow Card, after this step completes.
499
+
500
+ Note that you may add multiple `@card` decorators in a step with different parameters.
501
+
502
+
503
+ Parameters
504
+ ----------
505
+ type : str, default 'default'
506
+ Card type.
507
+ id : str, optional, default None
508
+ If multiple cards are present, use this id to identify this card.
509
+ options : Dict[str, Any], default {}
510
+ Options passed to the card. The contents depend on the card type.
511
+ timeout : int, default 45
512
+ Interrupt reporting if it takes more than this many seconds.
513
+ """
514
+ ...
515
+
516
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
517
+ """
518
+ Specifies that this step is used to deploy an instance of the app.
519
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
520
+
521
+
522
+ Parameters
523
+ ----------
524
+ app_port : int
525
+ Number of GPUs to use.
526
+ app_name : str
527
+ Name of the app to deploy.
528
+ """
529
+ ...
530
+
528
531
  def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
529
532
  """
530
533
  Decorator that helps cache, version and store models/datasets from huggingface hub.
@@ -551,78 +554,115 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
551
554
  ...
552
555
 
553
556
  @typing.overload
554
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
557
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
555
558
  """
556
- Enables loading / saving of models within a step.
557
-
558
-
559
-
560
- Parameters
561
- ----------
562
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
563
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
564
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
565
- - `current.checkpoint`
566
- - `current.model`
567
- - `current.huggingface_hub`
568
-
569
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
570
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
571
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
572
-
573
- temp_dir_root : str, default: None
574
- The root directory under which `current.model.loaded` will store loaded models
559
+ Decorator prototype for all step decorators. This function gets specialized
560
+ and imported for all decorators types by _import_plugin_decorators().
575
561
  """
576
562
  ...
577
563
 
578
564
  @typing.overload
579
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
565
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
580
566
  ...
581
567
 
582
- @typing.overload
583
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
568
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
569
+ """
570
+ Decorator prototype for all step decorators. This function gets specialized
571
+ and imported for all decorators types by _import_plugin_decorators().
572
+ """
584
573
  ...
585
574
 
586
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
575
+ @typing.overload
576
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
587
577
  """
588
- Enables loading / saving of models within a step.
578
+ Specifies the PyPI packages for the step.
589
579
 
580
+ Information in this decorator will augment any
581
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
582
+ you can use `@pypi_base` to set packages required by all
583
+ steps and use `@pypi` to specify step-specific overrides.
590
584
 
591
585
 
592
586
  Parameters
593
587
  ----------
594
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
595
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
596
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
597
- - `current.checkpoint`
598
- - `current.model`
599
- - `current.huggingface_hub`
600
-
601
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
602
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
603
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
604
-
605
- temp_dir_root : str, default: None
606
- The root directory under which `current.model.loaded` will store loaded models
588
+ packages : Dict[str, str], default: {}
589
+ Packages to use for this step. The key is the name of the package
590
+ and the value is the version to use.
591
+ python : str, optional, default: None
592
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
593
+ that the version used will correspond to the version of the Python interpreter used to start the run.
607
594
  """
608
595
  ...
609
596
 
610
597
  @typing.overload
611
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
612
- """
613
- Decorator prototype for all step decorators. This function gets specialized
614
- and imported for all decorators types by _import_plugin_decorators().
615
- """
598
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
616
599
  ...
617
600
 
618
601
  @typing.overload
619
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
602
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
620
603
  ...
621
604
 
622
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
605
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
623
606
  """
624
- Decorator prototype for all step decorators. This function gets specialized
625
- and imported for all decorators types by _import_plugin_decorators().
607
+ Specifies the PyPI packages for the step.
608
+
609
+ Information in this decorator will augment any
610
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
611
+ you can use `@pypi_base` to set packages required by all
612
+ steps and use `@pypi` to specify step-specific overrides.
613
+
614
+
615
+ Parameters
616
+ ----------
617
+ packages : Dict[str, str], default: {}
618
+ Packages to use for this step. The key is the name of the package
619
+ and the value is the version to use.
620
+ python : str, optional, default: None
621
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
622
+ that the version used will correspond to the version of the Python interpreter used to start the run.
623
+ """
624
+ ...
625
+
626
+ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy: str, force_cache_update: bool, debug: bool, circuit_breaker_config: dict, timeout_config: dict) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
627
+ """
628
+ This decorator is used to run Ollama APIs as Metaflow task sidecars.
629
+
630
+ User code call
631
+ --------------
632
+ @ollama(
633
+ models=[...],
634
+ ...
635
+ )
636
+
637
+ Valid backend options
638
+ ---------------------
639
+ - 'local': Run as a separate process on the local task machine.
640
+ - (TODO) 'managed': Outerbounds hosts and selects compute provider.
641
+ - (TODO) 'remote': Spin up separate instance to serve Ollama models.
642
+
643
+ Valid model options
644
+ -------------------
645
+ Any model here https://ollama.com/search, e.g. 'llama3.2', 'llama3.3'
646
+
647
+
648
+ Parameters
649
+ ----------
650
+ models: list[str]
651
+ List of Ollama containers running models in sidecars.
652
+ backend: str
653
+ Determines where and how to run the Ollama process.
654
+ force_pull: bool
655
+ Whether to run `ollama pull` no matter what, or first check the remote cache in Metaflow datastore for this model key.
656
+ cache_update_policy: str
657
+ Cache update policy: "auto", "force", or "never".
658
+ force_cache_update: bool
659
+ Simple override for "force" cache update policy.
660
+ debug: bool
661
+ Whether to turn on verbose debugging logs.
662
+ circuit_breaker_config: dict
663
+ Configuration for circuit breaker protection. Keys: failure_threshold, recovery_timeout, reset_timeout.
664
+ timeout_config: dict
665
+ Configuration for various operation timeouts. Keys: pull, stop, health_check, install, server_startup.
626
666
  """
627
667
  ...
628
668
 
@@ -686,35 +726,149 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
686
726
  ...
687
727
 
688
728
  @typing.overload
689
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
729
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
690
730
  """
691
- Specifies environment variables to be set prior to the execution of a step.
731
+ Specifies the number of times the task corresponding
732
+ to a step needs to be retried.
733
+
734
+ This decorator is useful for handling transient errors, such as networking issues.
735
+ If your task contains operations that can't be retried safely, e.g. database updates,
736
+ it is advisable to annotate it with `@retry(times=0)`.
737
+
738
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
739
+ decorator will execute a no-op task after all retries have been exhausted,
740
+ ensuring that the flow execution can continue.
692
741
 
693
742
 
694
743
  Parameters
695
744
  ----------
696
- vars : Dict[str, str], default {}
697
- Dictionary of environment variables to set.
745
+ times : int, default 3
746
+ Number of times to retry this task.
747
+ minutes_between_retries : int, default 2
748
+ Number of minutes between retries.
698
749
  """
699
750
  ...
700
751
 
701
752
  @typing.overload
702
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
753
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
703
754
  ...
704
755
 
705
756
  @typing.overload
706
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
757
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
707
758
  ...
708
759
 
709
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
760
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
710
761
  """
711
- Specifies environment variables to be set prior to the execution of a step.
762
+ Specifies the number of times the task corresponding
763
+ to a step needs to be retried.
764
+
765
+ This decorator is useful for handling transient errors, such as networking issues.
766
+ If your task contains operations that can't be retried safely, e.g. database updates,
767
+ it is advisable to annotate it with `@retry(times=0)`.
768
+
769
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
770
+ decorator will execute a no-op task after all retries have been exhausted,
771
+ ensuring that the flow execution can continue.
712
772
 
713
773
 
714
774
  Parameters
715
775
  ----------
716
- vars : Dict[str, str], default {}
717
- Dictionary of environment variables to set.
776
+ times : int, default 3
777
+ Number of times to retry this task.
778
+ minutes_between_retries : int, default 2
779
+ Number of minutes between retries.
780
+ """
781
+ ...
782
+
783
+ @typing.overload
784
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
785
+ """
786
+ Enables loading / saving of models within a step.
787
+
788
+
789
+
790
+ Parameters
791
+ ----------
792
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
793
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
794
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
795
+ - `current.checkpoint`
796
+ - `current.model`
797
+ - `current.huggingface_hub`
798
+
799
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
800
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
801
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
802
+
803
+ temp_dir_root : str, default: None
804
+ The root directory under which `current.model.loaded` will store loaded models
805
+ """
806
+ ...
807
+
808
+ @typing.overload
809
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
810
+ ...
811
+
812
+ @typing.overload
813
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
814
+ ...
815
+
816
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
817
+ """
818
+ Enables loading / saving of models within a step.
819
+
820
+
821
+
822
+ Parameters
823
+ ----------
824
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
825
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
826
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
827
+ - `current.checkpoint`
828
+ - `current.model`
829
+ - `current.huggingface_hub`
830
+
831
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
832
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
833
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
834
+
835
+ temp_dir_root : str, default: None
836
+ The root directory under which `current.model.loaded` will store loaded models
837
+ """
838
+ ...
839
+
840
+ @typing.overload
841
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
842
+ """
843
+ Specifies secrets to be retrieved and injected as environment variables prior to
844
+ the execution of a step.
845
+
846
+
847
+ Parameters
848
+ ----------
849
+ sources : List[Union[str, Dict[str, Any]]], default: []
850
+ List of secret specs, defining how the secrets are to be retrieved
851
+ """
852
+ ...
853
+
854
+ @typing.overload
855
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
856
+ ...
857
+
858
+ @typing.overload
859
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
860
+ ...
861
+
862
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
863
+ """
864
+ Specifies secrets to be retrieved and injected as environment variables prior to
865
+ the execution of a step.
866
+
867
+
868
+ Parameters
869
+ ----------
870
+ sources : List[Union[str, Dict[str, Any]]], default: []
871
+ List of secret specs, defining how the secrets are to be retrieved
718
872
  """
719
873
  ...
720
874
 
@@ -777,295 +931,192 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
777
931
  """
778
932
  ...
779
933
 
780
- @typing.overload
781
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
782
- """
783
- Specifies that the step will success under all circumstances.
784
-
785
- The decorator will create an optional artifact, specified by `var`, which
786
- contains the exception raised. You can use it to detect the presence
787
- of errors, indicating that all happy-path artifacts produced by the step
788
- are missing.
789
-
790
-
791
- Parameters
792
- ----------
793
- var : str, optional, default None
794
- Name of the artifact in which to store the caught exception.
795
- If not specified, the exception is not stored.
796
- print_exception : bool, default True
797
- Determines whether or not the exception is printed to
798
- stdout when caught.
799
- """
800
- ...
801
-
802
- @typing.overload
803
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
804
- ...
805
-
806
- @typing.overload
807
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
808
- ...
809
-
810
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
811
- """
812
- Specifies that the step will success under all circumstances.
813
-
814
- The decorator will create an optional artifact, specified by `var`, which
815
- contains the exception raised. You can use it to detect the presence
816
- of errors, indicating that all happy-path artifacts produced by the step
817
- are missing.
818
-
819
-
820
- Parameters
821
- ----------
822
- var : str, optional, default None
823
- Name of the artifact in which to store the caught exception.
824
- If not specified, the exception is not stored.
825
- print_exception : bool, default True
826
- Determines whether or not the exception is printed to
827
- stdout when caught.
828
- """
829
- ...
830
-
831
- @typing.overload
832
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
833
- """
834
- Specifies secrets to be retrieved and injected as environment variables prior to
835
- the execution of a step.
836
-
837
-
838
- Parameters
839
- ----------
840
- sources : List[Union[str, Dict[str, Any]]], default: []
841
- List of secret specs, defining how the secrets are to be retrieved
842
- """
843
- ...
844
-
845
- @typing.overload
846
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
847
- ...
848
-
849
- @typing.overload
850
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
851
- ...
852
-
853
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
934
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
854
935
  """
855
- Specifies secrets to be retrieved and injected as environment variables prior to
856
- the execution of a step.
936
+ Specifies that this step should execute on DGX cloud.
857
937
 
858
938
 
859
939
  Parameters
860
940
  ----------
861
- sources : List[Union[str, Dict[str, Any]]], default: []
862
- List of secret specs, defining how the secrets are to be retrieved
941
+ gpu : int
942
+ Number of GPUs to use.
943
+ gpu_type : str
944
+ Type of Nvidia GPU to use.
863
945
  """
864
946
  ...
865
947
 
866
948
  @typing.overload
867
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
949
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
868
950
  """
869
- Specifies the PyPI packages for the step.
870
-
871
- Information in this decorator will augment any
872
- attributes set in the `@pyi_base` flow-level decorator. Hence,
873
- you can use `@pypi_base` to set packages required by all
874
- steps and use `@pypi` to specify step-specific overrides.
875
-
876
-
877
- Parameters
878
- ----------
879
- packages : Dict[str, str], default: {}
880
- Packages to use for this step. The key is the name of the package
881
- and the value is the version to use.
882
- python : str, optional, default: None
883
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
884
- that the version used will correspond to the version of the Python interpreter used to start the run.
951
+ Internal decorator to support Fast bakery
885
952
  """
886
953
  ...
887
954
 
888
955
  @typing.overload
889
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
890
- ...
891
-
892
- @typing.overload
893
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
956
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
894
957
  ...
895
958
 
896
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
959
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
897
960
  """
898
- Specifies the PyPI packages for the step.
899
-
900
- Information in this decorator will augment any
901
- attributes set in the `@pyi_base` flow-level decorator. Hence,
902
- you can use `@pypi_base` to set packages required by all
903
- steps and use `@pypi` to specify step-specific overrides.
904
-
905
-
906
- Parameters
907
- ----------
908
- packages : Dict[str, str], default: {}
909
- Packages to use for this step. The key is the name of the package
910
- and the value is the version to use.
911
- python : str, optional, default: None
912
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
913
- that the version used will correspond to the version of the Python interpreter used to start the run.
961
+ Internal decorator to support Fast bakery
914
962
  """
915
963
  ...
916
964
 
917
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
965
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
918
966
  """
919
- Specifies that this step is used to deploy an instance of the app.
920
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
967
+ Specifies that this step should execute on DGX cloud.
921
968
 
922
969
 
923
970
  Parameters
924
971
  ----------
925
- app_port : int
972
+ gpu : int
926
973
  Number of GPUs to use.
927
- app_name : str
928
- Name of the app to deploy.
929
- """
930
- ...
931
-
932
- @typing.overload
933
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
934
- """
935
- Creates a human-readable report, a Metaflow Card, after this step completes.
936
-
937
- Note that you may add multiple `@card` decorators in a step with different parameters.
938
-
939
-
940
- Parameters
941
- ----------
942
- type : str, default 'default'
943
- Card type.
944
- id : str, optional, default None
945
- If multiple cards are present, use this id to identify this card.
946
- options : Dict[str, Any], default {}
947
- Options passed to the card. The contents depend on the card type.
948
- timeout : int, default 45
949
- Interrupt reporting if it takes more than this many seconds.
974
+ gpu_type : str
975
+ Type of Nvidia GPU to use.
976
+ queue_timeout : int
977
+ Time to keep the job in NVCF's queue.
950
978
  """
951
979
  ...
952
980
 
953
- @typing.overload
954
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
955
- ...
956
-
957
- @typing.overload
958
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
959
- ...
960
-
961
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
981
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
962
982
  """
963
- Creates a human-readable report, a Metaflow Card, after this step completes.
964
-
965
- Note that you may add multiple `@card` decorators in a step with different parameters.
983
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
984
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
966
985
 
967
986
 
968
987
  Parameters
969
988
  ----------
970
- type : str, default 'default'
971
- Card type.
972
- id : str, optional, default None
973
- If multiple cards are present, use this id to identify this card.
974
- options : Dict[str, Any], default {}
975
- Options passed to the card. The contents depend on the card type.
976
- timeout : int, default 45
977
- Interrupt reporting if it takes more than this many seconds.
989
+ timeout : int
990
+ Time, in seconds before the task times out and fails. (Default: 3600)
991
+ poke_interval : int
992
+ Time in seconds that the job should wait in between each try. (Default: 60)
993
+ mode : str
994
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
995
+ exponential_backoff : bool
996
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
997
+ pool : str
998
+ the slot pool this task should run in,
999
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1000
+ soft_fail : bool
1001
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1002
+ name : str
1003
+ Name of the sensor on Airflow
1004
+ description : str
1005
+ Description of sensor in the Airflow UI
1006
+ external_dag_id : str
1007
+ The dag_id that contains the task you want to wait for.
1008
+ external_task_ids : List[str]
1009
+ The list of task_ids that you want to wait for.
1010
+ If None (default value) the sensor waits for the DAG. (Default: None)
1011
+ allowed_states : List[str]
1012
+ Iterable of allowed states, (Default: ['success'])
1013
+ failed_states : List[str]
1014
+ Iterable of failed or dis-allowed states. (Default: None)
1015
+ execution_delta : datetime.timedelta
1016
+ time difference with the previous execution to look at,
1017
+ the default is the same logical date as the current task or DAG. (Default: None)
1018
+ check_existence: bool
1019
+ Set to True to check if the external task exists or check if
1020
+ the DAG to wait for exists. (Default: True)
978
1021
  """
979
1022
  ...
980
1023
 
981
1024
  @typing.overload
982
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1025
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
983
1026
  """
984
- Specifies the event(s) that this flow depends on.
1027
+ Specifies the flow(s) that this flow depends on.
985
1028
 
986
1029
  ```
987
- @trigger(event='foo')
1030
+ @trigger_on_finish(flow='FooFlow')
988
1031
  ```
989
1032
  or
990
1033
  ```
991
- @trigger(events=['foo', 'bar'])
1034
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
992
1035
  ```
1036
+ This decorator respects the @project decorator and triggers the flow
1037
+ when upstream runs within the same namespace complete successfully
993
1038
 
994
- Additionally, you can specify the parameter mappings
995
- to map event payload to Metaflow parameters for the flow.
1039
+ Additionally, you can specify project aware upstream flow dependencies
1040
+ by specifying the fully qualified project_flow_name.
996
1041
  ```
997
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1042
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
998
1043
  ```
999
1044
  or
1000
1045
  ```
1001
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1002
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1046
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1003
1047
  ```
1004
1048
 
1005
- 'parameters' can also be a list of strings and tuples like so:
1006
- ```
1007
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1008
- ```
1009
- This is equivalent to:
1049
+ You can also specify just the project or project branch (other values will be
1050
+ inferred from the current project or project branch):
1010
1051
  ```
1011
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1052
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1012
1053
  ```
1013
1054
 
1055
+ Note that `branch` is typically one of:
1056
+ - `prod`
1057
+ - `user.bob`
1058
+ - `test.my_experiment`
1059
+ - `prod.staging`
1060
+
1014
1061
 
1015
1062
  Parameters
1016
1063
  ----------
1017
- event : Union[str, Dict[str, Any]], optional, default None
1018
- Event dependency for this flow.
1019
- events : List[Union[str, Dict[str, Any]]], default []
1020
- Events dependency for this flow.
1064
+ flow : Union[str, Dict[str, str]], optional, default None
1065
+ Upstream flow dependency for this flow.
1066
+ flows : List[Union[str, Dict[str, str]]], default []
1067
+ Upstream flow dependencies for this flow.
1021
1068
  options : Dict[str, Any], default {}
1022
1069
  Backend-specific configuration for tuning eventing behavior.
1023
1070
  """
1024
1071
  ...
1025
1072
 
1026
1073
  @typing.overload
1027
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1074
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1028
1075
  ...
1029
1076
 
1030
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1077
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1031
1078
  """
1032
- Specifies the event(s) that this flow depends on.
1079
+ Specifies the flow(s) that this flow depends on.
1033
1080
 
1034
1081
  ```
1035
- @trigger(event='foo')
1082
+ @trigger_on_finish(flow='FooFlow')
1036
1083
  ```
1037
1084
  or
1038
1085
  ```
1039
- @trigger(events=['foo', 'bar'])
1086
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1040
1087
  ```
1088
+ This decorator respects the @project decorator and triggers the flow
1089
+ when upstream runs within the same namespace complete successfully
1041
1090
 
1042
- Additionally, you can specify the parameter mappings
1043
- to map event payload to Metaflow parameters for the flow.
1091
+ Additionally, you can specify project aware upstream flow dependencies
1092
+ by specifying the fully qualified project_flow_name.
1044
1093
  ```
1045
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1094
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1046
1095
  ```
1047
1096
  or
1048
1097
  ```
1049
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1050
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1098
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1051
1099
  ```
1052
1100
 
1053
- 'parameters' can also be a list of strings and tuples like so:
1054
- ```
1055
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1056
- ```
1057
- This is equivalent to:
1101
+ You can also specify just the project or project branch (other values will be
1102
+ inferred from the current project or project branch):
1058
1103
  ```
1059
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1104
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1060
1105
  ```
1061
1106
 
1107
+ Note that `branch` is typically one of:
1108
+ - `prod`
1109
+ - `user.bob`
1110
+ - `test.my_experiment`
1111
+ - `prod.staging`
1112
+
1062
1113
 
1063
1114
  Parameters
1064
1115
  ----------
1065
- event : Union[str, Dict[str, Any]], optional, default None
1066
- Event dependency for this flow.
1067
- events : List[Union[str, Dict[str, Any]]], default []
1068
- Events dependency for this flow.
1116
+ flow : Union[str, Dict[str, str]], optional, default None
1117
+ Upstream flow dependency for this flow.
1118
+ flows : List[Union[str, Dict[str, str]]], default []
1119
+ Upstream flow dependencies for this flow.
1069
1120
  options : Dict[str, Any], default {}
1070
1121
  Backend-specific configuration for tuning eventing behavior.
1071
1122
  """
@@ -1122,158 +1173,234 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1122
1173
  """
1123
1174
  ...
1124
1175
 
1176
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1177
+ """
1178
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1179
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1180
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1181
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1182
+ starts only after all sensors finish.
1183
+
1184
+
1185
+ Parameters
1186
+ ----------
1187
+ timeout : int
1188
+ Time, in seconds before the task times out and fails. (Default: 3600)
1189
+ poke_interval : int
1190
+ Time in seconds that the job should wait in between each try. (Default: 60)
1191
+ mode : str
1192
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1193
+ exponential_backoff : bool
1194
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1195
+ pool : str
1196
+ the slot pool this task should run in,
1197
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1198
+ soft_fail : bool
1199
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1200
+ name : str
1201
+ Name of the sensor on Airflow
1202
+ description : str
1203
+ Description of sensor in the Airflow UI
1204
+ bucket_key : Union[str, List[str]]
1205
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1206
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1207
+ bucket_name : str
1208
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1209
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1210
+ wildcard_match : bool
1211
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1212
+ aws_conn_id : str
1213
+ a reference to the s3 connection on Airflow. (Default: None)
1214
+ verify : bool
1215
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1216
+ """
1217
+ ...
1218
+
1125
1219
  @typing.overload
1126
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1220
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1127
1221
  """
1128
- Specifies the flow(s) that this flow depends on.
1222
+ Specifies the times when the flow should be run when running on a
1223
+ production scheduler.
1224
+
1225
+
1226
+ Parameters
1227
+ ----------
1228
+ hourly : bool, default False
1229
+ Run the workflow hourly.
1230
+ daily : bool, default True
1231
+ Run the workflow daily.
1232
+ weekly : bool, default False
1233
+ Run the workflow weekly.
1234
+ cron : str, optional, default None
1235
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1236
+ specified by this expression.
1237
+ timezone : str, optional, default None
1238
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1239
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1240
+ """
1241
+ ...
1242
+
1243
+ @typing.overload
1244
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1245
+ ...
1246
+
1247
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1248
+ """
1249
+ Specifies the times when the flow should be run when running on a
1250
+ production scheduler.
1251
+
1252
+
1253
+ Parameters
1254
+ ----------
1255
+ hourly : bool, default False
1256
+ Run the workflow hourly.
1257
+ daily : bool, default True
1258
+ Run the workflow daily.
1259
+ weekly : bool, default False
1260
+ Run the workflow weekly.
1261
+ cron : str, optional, default None
1262
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1263
+ specified by this expression.
1264
+ timezone : str, optional, default None
1265
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1266
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1267
+ """
1268
+ ...
1269
+
1270
+ @typing.overload
1271
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1272
+ """
1273
+ Specifies the PyPI packages for all steps of the flow.
1274
+
1275
+ Use `@pypi_base` to set common packages required by all
1276
+ steps and use `@pypi` to specify step-specific overrides.
1277
+
1278
+ Parameters
1279
+ ----------
1280
+ packages : Dict[str, str], default: {}
1281
+ Packages to use for this flow. The key is the name of the package
1282
+ and the value is the version to use.
1283
+ python : str, optional, default: None
1284
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1285
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1286
+ """
1287
+ ...
1288
+
1289
+ @typing.overload
1290
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1291
+ ...
1292
+
1293
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1294
+ """
1295
+ Specifies the PyPI packages for all steps of the flow.
1296
+
1297
+ Use `@pypi_base` to set common packages required by all
1298
+ steps and use `@pypi` to specify step-specific overrides.
1299
+
1300
+ Parameters
1301
+ ----------
1302
+ packages : Dict[str, str], default: {}
1303
+ Packages to use for this flow. The key is the name of the package
1304
+ and the value is the version to use.
1305
+ python : str, optional, default: None
1306
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1307
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1308
+ """
1309
+ ...
1310
+
1311
+ @typing.overload
1312
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1313
+ """
1314
+ Specifies the event(s) that this flow depends on.
1129
1315
 
1130
1316
  ```
1131
- @trigger_on_finish(flow='FooFlow')
1317
+ @trigger(event='foo')
1132
1318
  ```
1133
1319
  or
1134
1320
  ```
1135
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1321
+ @trigger(events=['foo', 'bar'])
1136
1322
  ```
1137
- This decorator respects the @project decorator and triggers the flow
1138
- when upstream runs within the same namespace complete successfully
1139
1323
 
1140
- Additionally, you can specify project aware upstream flow dependencies
1141
- by specifying the fully qualified project_flow_name.
1324
+ Additionally, you can specify the parameter mappings
1325
+ to map event payload to Metaflow parameters for the flow.
1142
1326
  ```
1143
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1327
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1144
1328
  ```
1145
1329
  or
1146
1330
  ```
1147
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1331
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1332
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1148
1333
  ```
1149
1334
 
1150
- You can also specify just the project or project branch (other values will be
1151
- inferred from the current project or project branch):
1335
+ 'parameters' can also be a list of strings and tuples like so:
1152
1336
  ```
1153
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1337
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1338
+ ```
1339
+ This is equivalent to:
1340
+ ```
1341
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1154
1342
  ```
1155
-
1156
- Note that `branch` is typically one of:
1157
- - `prod`
1158
- - `user.bob`
1159
- - `test.my_experiment`
1160
- - `prod.staging`
1161
1343
 
1162
1344
 
1163
1345
  Parameters
1164
1346
  ----------
1165
- flow : Union[str, Dict[str, str]], optional, default None
1166
- Upstream flow dependency for this flow.
1167
- flows : List[Union[str, Dict[str, str]]], default []
1168
- Upstream flow dependencies for this flow.
1347
+ event : Union[str, Dict[str, Any]], optional, default None
1348
+ Event dependency for this flow.
1349
+ events : List[Union[str, Dict[str, Any]]], default []
1350
+ Events dependency for this flow.
1169
1351
  options : Dict[str, Any], default {}
1170
1352
  Backend-specific configuration for tuning eventing behavior.
1171
1353
  """
1172
1354
  ...
1173
1355
 
1174
1356
  @typing.overload
1175
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1357
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1176
1358
  ...
1177
1359
 
1178
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1360
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1179
1361
  """
1180
- Specifies the flow(s) that this flow depends on.
1362
+ Specifies the event(s) that this flow depends on.
1181
1363
 
1182
1364
  ```
1183
- @trigger_on_finish(flow='FooFlow')
1365
+ @trigger(event='foo')
1184
1366
  ```
1185
1367
  or
1186
1368
  ```
1187
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1369
+ @trigger(events=['foo', 'bar'])
1188
1370
  ```
1189
- This decorator respects the @project decorator and triggers the flow
1190
- when upstream runs within the same namespace complete successfully
1191
1371
 
1192
- Additionally, you can specify project aware upstream flow dependencies
1193
- by specifying the fully qualified project_flow_name.
1372
+ Additionally, you can specify the parameter mappings
1373
+ to map event payload to Metaflow parameters for the flow.
1194
1374
  ```
1195
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1375
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1196
1376
  ```
1197
1377
  or
1198
1378
  ```
1199
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1379
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1380
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1200
1381
  ```
1201
1382
 
1202
- You can also specify just the project or project branch (other values will be
1203
- inferred from the current project or project branch):
1383
+ 'parameters' can also be a list of strings and tuples like so:
1204
1384
  ```
1205
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1385
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1386
+ ```
1387
+ This is equivalent to:
1388
+ ```
1389
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1206
1390
  ```
1207
-
1208
- Note that `branch` is typically one of:
1209
- - `prod`
1210
- - `user.bob`
1211
- - `test.my_experiment`
1212
- - `prod.staging`
1213
1391
 
1214
1392
 
1215
1393
  Parameters
1216
1394
  ----------
1217
- flow : Union[str, Dict[str, str]], optional, default None
1218
- Upstream flow dependency for this flow.
1219
- flows : List[Union[str, Dict[str, str]]], default []
1220
- Upstream flow dependencies for this flow.
1395
+ event : Union[str, Dict[str, Any]], optional, default None
1396
+ Event dependency for this flow.
1397
+ events : List[Union[str, Dict[str, Any]]], default []
1398
+ Events dependency for this flow.
1221
1399
  options : Dict[str, Any], default {}
1222
1400
  Backend-specific configuration for tuning eventing behavior.
1223
1401
  """
1224
1402
  ...
1225
1403
 
1226
- @typing.overload
1227
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1228
- """
1229
- Specifies the times when the flow should be run when running on a
1230
- production scheduler.
1231
-
1232
-
1233
- Parameters
1234
- ----------
1235
- hourly : bool, default False
1236
- Run the workflow hourly.
1237
- daily : bool, default True
1238
- Run the workflow daily.
1239
- weekly : bool, default False
1240
- Run the workflow weekly.
1241
- cron : str, optional, default None
1242
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1243
- specified by this expression.
1244
- timezone : str, optional, default None
1245
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1246
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1247
- """
1248
- ...
1249
-
1250
- @typing.overload
1251
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1252
- ...
1253
-
1254
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1255
- """
1256
- Specifies the times when the flow should be run when running on a
1257
- production scheduler.
1258
-
1259
-
1260
- Parameters
1261
- ----------
1262
- hourly : bool, default False
1263
- Run the workflow hourly.
1264
- daily : bool, default True
1265
- Run the workflow daily.
1266
- weekly : bool, default False
1267
- Run the workflow weekly.
1268
- cron : str, optional, default None
1269
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1270
- specified by this expression.
1271
- timezone : str, optional, default None
1272
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1273
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1274
- """
1275
- ...
1276
-
1277
1404
  def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1278
1405
  """
1279
1406
  Allows setting external datastores to save data for the
@@ -1388,47 +1515,6 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1388
1515
  """
1389
1516
  ...
1390
1517
 
1391
- @typing.overload
1392
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1393
- """
1394
- Specifies the PyPI packages for all steps of the flow.
1395
-
1396
- Use `@pypi_base` to set common packages required by all
1397
- steps and use `@pypi` to specify step-specific overrides.
1398
-
1399
- Parameters
1400
- ----------
1401
- packages : Dict[str, str], default: {}
1402
- Packages to use for this flow. The key is the name of the package
1403
- and the value is the version to use.
1404
- python : str, optional, default: None
1405
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1406
- that the version used will correspond to the version of the Python interpreter used to start the run.
1407
- """
1408
- ...
1409
-
1410
- @typing.overload
1411
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1412
- ...
1413
-
1414
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1415
- """
1416
- Specifies the PyPI packages for all steps of the flow.
1417
-
1418
- Use `@pypi_base` to set common packages required by all
1419
- steps and use `@pypi` to specify step-specific overrides.
1420
-
1421
- Parameters
1422
- ----------
1423
- packages : Dict[str, str], default: {}
1424
- Packages to use for this flow. The key is the name of the package
1425
- and the value is the version to use.
1426
- python : str, optional, default: None
1427
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1428
- that the version used will correspond to the version of the Python interpreter used to start the run.
1429
- """
1430
- ...
1431
-
1432
1518
  def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1433
1519
  """
1434
1520
  Specifies what flows belong to the same project.
@@ -1464,91 +1550,5 @@ def project(*, name: str, branch: typing.Optional[str] = None, production: bool
1464
1550
  """
1465
1551
  ...
1466
1552
 
1467
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1468
- """
1469
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1470
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1471
-
1472
-
1473
- Parameters
1474
- ----------
1475
- timeout : int
1476
- Time, in seconds before the task times out and fails. (Default: 3600)
1477
- poke_interval : int
1478
- Time in seconds that the job should wait in between each try. (Default: 60)
1479
- mode : str
1480
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1481
- exponential_backoff : bool
1482
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1483
- pool : str
1484
- the slot pool this task should run in,
1485
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1486
- soft_fail : bool
1487
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1488
- name : str
1489
- Name of the sensor on Airflow
1490
- description : str
1491
- Description of sensor in the Airflow UI
1492
- external_dag_id : str
1493
- The dag_id that contains the task you want to wait for.
1494
- external_task_ids : List[str]
1495
- The list of task_ids that you want to wait for.
1496
- If None (default value) the sensor waits for the DAG. (Default: None)
1497
- allowed_states : List[str]
1498
- Iterable of allowed states, (Default: ['success'])
1499
- failed_states : List[str]
1500
- Iterable of failed or dis-allowed states. (Default: None)
1501
- execution_delta : datetime.timedelta
1502
- time difference with the previous execution to look at,
1503
- the default is the same logical date as the current task or DAG. (Default: None)
1504
- check_existence: bool
1505
- Set to True to check if the external task exists or check if
1506
- the DAG to wait for exists. (Default: True)
1507
- """
1508
- ...
1509
-
1510
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1511
- """
1512
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1513
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1514
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1515
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1516
- starts only after all sensors finish.
1517
-
1518
-
1519
- Parameters
1520
- ----------
1521
- timeout : int
1522
- Time, in seconds before the task times out and fails. (Default: 3600)
1523
- poke_interval : int
1524
- Time in seconds that the job should wait in between each try. (Default: 60)
1525
- mode : str
1526
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1527
- exponential_backoff : bool
1528
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1529
- pool : str
1530
- the slot pool this task should run in,
1531
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1532
- soft_fail : bool
1533
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1534
- name : str
1535
- Name of the sensor on Airflow
1536
- description : str
1537
- Description of sensor in the Airflow UI
1538
- bucket_key : Union[str, List[str]]
1539
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1540
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1541
- bucket_name : str
1542
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1543
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1544
- wildcard_match : bool
1545
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1546
- aws_conn_id : str
1547
- a reference to the s3 connection on Airflow. (Default: None)
1548
- verify : bool
1549
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1550
- """
1551
- ...
1552
-
1553
1553
  pkg_name: str
1554
1554