ob-metaflow-stubs 6.0.3.179rc3__py2.py3-none-any.whl → 6.0.3.179rc4__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. metaflow-stubs/__init__.pyi +813 -813
  2. metaflow-stubs/cards.pyi +1 -1
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/cli_components/__init__.pyi +1 -1
  5. metaflow-stubs/cli_components/utils.pyi +1 -1
  6. metaflow-stubs/client/__init__.pyi +1 -1
  7. metaflow-stubs/client/core.pyi +4 -4
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +1 -1
  10. metaflow-stubs/exception.pyi +1 -1
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +2 -2
  14. metaflow-stubs/info_file.pyi +1 -1
  15. metaflow-stubs/metadata_provider/__init__.pyi +1 -1
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +1 -1
  17. metaflow-stubs/metadata_provider/metadata.pyi +1 -1
  18. metaflow-stubs/metadata_provider/util.pyi +1 -1
  19. metaflow-stubs/metaflow_config.pyi +1 -1
  20. metaflow-stubs/metaflow_current.pyi +108 -108
  21. metaflow-stubs/metaflow_git.pyi +1 -1
  22. metaflow-stubs/mf_extensions/__init__.pyi +1 -1
  23. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +1 -1
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +1 -1
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +1 -1
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +1 -1
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +1 -1
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +1 -1
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +1 -1
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +1 -1
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +1 -1
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +1 -1
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +3 -3
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +1 -1
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +1 -1
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +1 -1
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +3 -3
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +1 -1
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +1 -1
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +1 -1
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +1 -1
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +1 -1
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +1 -1
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +1 -1
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +1 -1
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +1 -1
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +3 -3
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +1 -1
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +1 -1
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +1 -1
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +1 -1
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +1 -1
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +1 -1
  63. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +1 -1
  64. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +1 -1
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +1 -1
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/__init__.pyi +1 -1
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/aws/assume_role_decorator.pyi +1 -1
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +1 -1
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/async_cards.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +1 -1
  71. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/__init__.pyi +1 -1
  72. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/coreweave.pyi +1 -1
  73. metaflow-stubs/mf_extensions/outerbounds/plugins/checkpoint_datastores/nebius.pyi +1 -1
  74. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/__init__.pyi +1 -1
  75. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/baker.pyi +3 -3
  76. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/docker_environment.pyi +1 -1
  77. metaflow-stubs/mf_extensions/outerbounds/plugins/fast_bakery/fast_bakery.pyi +1 -1
  78. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/__init__.pyi +1 -1
  79. metaflow-stubs/mf_extensions/outerbounds/plugins/kubernetes/pod_killer.pyi +1 -1
  80. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +1 -1
  81. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/constants.pyi +1 -1
  82. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/exceptions.pyi +1 -1
  83. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +1 -1
  84. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/status_card.pyi +1 -1
  85. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +1 -1
  86. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +1 -1
  87. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +1 -1
  88. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +1 -1
  89. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +1 -1
  90. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +1 -1
  91. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +1 -1
  92. metaflow-stubs/multicore_utils.pyi +1 -1
  93. metaflow-stubs/ob_internal.pyi +1 -1
  94. metaflow-stubs/parameters.pyi +2 -2
  95. metaflow-stubs/plugins/__init__.pyi +12 -12
  96. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  97. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  98. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  99. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  100. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  101. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +1 -1
  102. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +1 -1
  103. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  104. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  105. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  106. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  107. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  109. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +1 -1
  110. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  111. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  112. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  113. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  114. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  115. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  116. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  117. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  118. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +1 -1
  119. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  120. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  121. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  122. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  123. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  124. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  125. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  126. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  127. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  128. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  129. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +1 -1
  130. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  131. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  132. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  133. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  134. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  135. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  136. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  137. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  138. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  139. metaflow-stubs/plugins/cards/card_modules/basic.pyi +1 -1
  140. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  141. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  142. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  143. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  144. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +1 -1
  145. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  146. metaflow-stubs/plugins/cards/component_serializer.pyi +1 -1
  147. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  148. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  149. metaflow-stubs/plugins/datatools/__init__.pyi +1 -1
  150. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  151. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  152. metaflow-stubs/plugins/datatools/s3/s3.pyi +2 -2
  153. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  154. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  155. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  156. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  157. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  158. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  159. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  160. metaflow-stubs/plugins/frameworks/pytorch.pyi +1 -1
  161. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  162. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +1 -1
  163. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  164. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  165. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  166. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  167. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  168. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +1 -1
  169. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  170. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  171. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  172. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  173. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +1 -1
  174. metaflow-stubs/plugins/ollama/__init__.pyi +2 -2
  175. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  176. metaflow-stubs/plugins/perimeters.pyi +1 -1
  177. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  178. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  179. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  180. metaflow-stubs/plugins/pypi/conda_environment.pyi +2 -2
  181. metaflow-stubs/plugins/pypi/parsers.pyi +1 -1
  182. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  183. metaflow-stubs/plugins/pypi/pypi_environment.pyi +1 -1
  184. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  185. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  186. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  187. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  188. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
  189. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  190. metaflow-stubs/plugins/snowflake/__init__.pyi +1 -1
  191. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  192. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +1 -1
  193. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  194. metaflow-stubs/plugins/torchtune/__init__.pyi +1 -1
  195. metaflow-stubs/plugins/uv/__init__.pyi +1 -1
  196. metaflow-stubs/plugins/uv/uv_environment.pyi +1 -1
  197. metaflow-stubs/profilers/__init__.pyi +1 -1
  198. metaflow-stubs/pylint_wrapper.pyi +1 -1
  199. metaflow-stubs/runner/__init__.pyi +1 -1
  200. metaflow-stubs/runner/deployer.pyi +4 -4
  201. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  202. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  203. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  204. metaflow-stubs/runner/nbrun.pyi +1 -1
  205. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  206. metaflow-stubs/runner/utils.pyi +2 -2
  207. metaflow-stubs/system/__init__.pyi +1 -1
  208. metaflow-stubs/system/system_logger.pyi +2 -2
  209. metaflow-stubs/system/system_monitor.pyi +1 -1
  210. metaflow-stubs/tagging_util.pyi +1 -1
  211. metaflow-stubs/tuple_util.pyi +1 -1
  212. metaflow-stubs/user_configs/__init__.pyi +1 -1
  213. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  214. metaflow-stubs/user_configs/config_options.pyi +1 -1
  215. metaflow-stubs/user_configs/config_parameters.pyi +4 -4
  216. {ob_metaflow_stubs-6.0.3.179rc3.dist-info → ob_metaflow_stubs-6.0.3.179rc4.dist-info}/METADATA +1 -1
  217. ob_metaflow_stubs-6.0.3.179rc4.dist-info/RECORD +220 -0
  218. ob_metaflow_stubs-6.0.3.179rc3.dist-info/RECORD +0 -220
  219. {ob_metaflow_stubs-6.0.3.179rc3.dist-info → ob_metaflow_stubs-6.0.3.179rc4.dist-info}/WHEEL +0 -0
  220. {ob_metaflow_stubs-6.0.3.179rc3.dist-info → ob_metaflow_stubs-6.0.3.179rc4.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.15.17.1+obcheckpoint(0.2.1);ob(v1) #
4
- # Generated on 2025-06-13T20:43:51.355806 #
4
+ # Generated on 2025-06-13T21:20:48.635887 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -35,17 +35,17 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
- from . import tuple_util as tuple_util
39
38
  from . import cards as cards
40
- from . import events as events
41
39
  from . import metaflow_git as metaflow_git
40
+ from . import tuple_util as tuple_util
41
+ from . import events as events
42
42
  from . import runner as runner
43
43
  from . import plugins as plugins
44
44
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
45
45
  from . import includefile as includefile
46
46
  from .includefile import IncludeFile as IncludeFile
47
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
48
47
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
48
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
49
49
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
50
50
  from . import client as client
51
51
  from .client.core import namespace as namespace
@@ -156,217 +156,116 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
156
156
  ...
157
157
 
158
158
  @typing.overload
159
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
159
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
160
160
  """
161
- Enables checkpointing for a step.
161
+ Specifies the resources needed when executing this step.
162
+
163
+ Use `@resources` to specify the resource requirements
164
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
162
165
 
166
+ You can choose the compute layer on the command line by executing e.g.
167
+ ```
168
+ python myflow.py run --with batch
169
+ ```
170
+ or
171
+ ```
172
+ python myflow.py run --with kubernetes
173
+ ```
174
+ which executes the flow on the desired system using the
175
+ requirements specified in `@resources`.
163
176
 
164
177
 
165
178
  Parameters
166
179
  ----------
167
- load_policy : str, default: "fresh"
168
- The policy for loading the checkpoint. The following policies are supported:
169
- - "eager": Loads the the latest available checkpoint within the namespace.
170
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
171
- will be loaded at the start of the task.
172
- - "none": Do not load any checkpoint
173
- - "fresh": Loads the lastest checkpoint created within the running Task.
174
- This mode helps loading checkpoints across various retry attempts of the same task.
175
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
176
- created within the task will be loaded when the task is retries execution on failure.
177
-
178
- temp_dir_root : str, default: None
179
- The root directory under which `current.checkpoint.directory` will be created.
180
+ cpu : int, default 1
181
+ Number of CPUs required for this step.
182
+ gpu : int, optional, default None
183
+ Number of GPUs required for this step.
184
+ disk : int, optional, default None
185
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
186
+ memory : int, default 4096
187
+ Memory size (in MB) required for this step.
188
+ shared_memory : int, optional, default None
189
+ The value for the size (in MiB) of the /dev/shm volume for this step.
190
+ This parameter maps to the `--shm-size` option in Docker.
180
191
  """
181
192
  ...
182
193
 
183
194
  @typing.overload
184
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
195
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
185
196
  ...
186
197
 
187
198
  @typing.overload
188
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
199
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
189
200
  ...
190
201
 
191
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
202
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
192
203
  """
193
- Enables checkpointing for a step.
204
+ Specifies the resources needed when executing this step.
205
+
206
+ Use `@resources` to specify the resource requirements
207
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
194
208
 
209
+ You can choose the compute layer on the command line by executing e.g.
210
+ ```
211
+ python myflow.py run --with batch
212
+ ```
213
+ or
214
+ ```
215
+ python myflow.py run --with kubernetes
216
+ ```
217
+ which executes the flow on the desired system using the
218
+ requirements specified in `@resources`.
195
219
 
196
220
 
197
221
  Parameters
198
222
  ----------
199
- load_policy : str, default: "fresh"
200
- The policy for loading the checkpoint. The following policies are supported:
201
- - "eager": Loads the the latest available checkpoint within the namespace.
202
- With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
203
- will be loaded at the start of the task.
204
- - "none": Do not load any checkpoint
205
- - "fresh": Loads the lastest checkpoint created within the running Task.
206
- This mode helps loading checkpoints across various retry attempts of the same task.
207
- With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
208
- created within the task will be loaded when the task is retries execution on failure.
209
-
210
- temp_dir_root : str, default: None
211
- The root directory under which `current.checkpoint.directory` will be created.
223
+ cpu : int, default 1
224
+ Number of CPUs required for this step.
225
+ gpu : int, optional, default None
226
+ Number of GPUs required for this step.
227
+ disk : int, optional, default None
228
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
229
+ memory : int, default 4096
230
+ Memory size (in MB) required for this step.
231
+ shared_memory : int, optional, default None
232
+ The value for the size (in MiB) of the /dev/shm volume for this step.
233
+ This parameter maps to the `--shm-size` option in Docker.
212
234
  """
213
235
  ...
214
236
 
215
237
  @typing.overload
216
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
238
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
217
239
  """
218
- Specifies the number of times the task corresponding
219
- to a step needs to be retried.
220
-
221
- This decorator is useful for handling transient errors, such as networking issues.
222
- If your task contains operations that can't be retried safely, e.g. database updates,
223
- it is advisable to annotate it with `@retry(times=0)`.
224
-
225
- This can be used in conjunction with the `@catch` decorator. The `@catch`
226
- decorator will execute a no-op task after all retries have been exhausted,
227
- ensuring that the flow execution can continue.
240
+ Specifies secrets to be retrieved and injected as environment variables prior to
241
+ the execution of a step.
228
242
 
229
243
 
230
244
  Parameters
231
245
  ----------
232
- times : int, default 3
233
- Number of times to retry this task.
234
- minutes_between_retries : int, default 2
235
- Number of minutes between retries.
246
+ sources : List[Union[str, Dict[str, Any]]], default: []
247
+ List of secret specs, defining how the secrets are to be retrieved
236
248
  """
237
249
  ...
238
250
 
239
251
  @typing.overload
240
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
252
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
241
253
  ...
242
254
 
243
255
  @typing.overload
244
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
245
- ...
246
-
247
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
248
- """
249
- Specifies the number of times the task corresponding
250
- to a step needs to be retried.
251
-
252
- This decorator is useful for handling transient errors, such as networking issues.
253
- If your task contains operations that can't be retried safely, e.g. database updates,
254
- it is advisable to annotate it with `@retry(times=0)`.
255
-
256
- This can be used in conjunction with the `@catch` decorator. The `@catch`
257
- decorator will execute a no-op task after all retries have been exhausted,
258
- ensuring that the flow execution can continue.
259
-
260
-
261
- Parameters
262
- ----------
263
- times : int, default 3
264
- Number of times to retry this task.
265
- minutes_between_retries : int, default 2
266
- Number of minutes between retries.
267
- """
268
- ...
269
-
270
- def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
271
- """
272
- Specifies that this step should execute on DGX cloud.
273
-
274
-
275
- Parameters
276
- ----------
277
- gpu : int
278
- Number of GPUs to use.
279
- gpu_type : str
280
- Type of Nvidia GPU to use.
281
- """
256
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
282
257
  ...
283
258
 
284
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
259
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
285
260
  """
286
- Specifies that this step should execute on Kubernetes.
261
+ Specifies secrets to be retrieved and injected as environment variables prior to
262
+ the execution of a step.
287
263
 
288
264
 
289
265
  Parameters
290
266
  ----------
291
- cpu : int, default 1
292
- Number of CPUs required for this step. If `@resources` is
293
- also present, the maximum value from all decorators is used.
294
- memory : int, default 4096
295
- Memory size (in MB) required for this step. If
296
- `@resources` is also present, the maximum value from all decorators is
297
- used.
298
- disk : int, default 10240
299
- Disk size (in MB) required for this step. If
300
- `@resources` is also present, the maximum value from all decorators is
301
- used.
302
- image : str, optional, default None
303
- Docker image to use when launching on Kubernetes. If not specified, and
304
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
305
- not, a default Docker image mapping to the current version of Python is used.
306
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
307
- If given, the imagePullPolicy to be applied to the Docker image of the step.
308
- image_pull_secrets: List[str], default []
309
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
310
- Kubernetes image pull secrets to use when pulling container images
311
- in Kubernetes.
312
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
313
- Kubernetes service account to use when launching pod in Kubernetes.
314
- secrets : List[str], optional, default None
315
- Kubernetes secrets to use when launching pod in Kubernetes. These
316
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
317
- in Metaflow configuration.
318
- node_selector: Union[Dict[str,str], str], optional, default None
319
- Kubernetes node selector(s) to apply to the pod running the task.
320
- Can be passed in as a comma separated string of values e.g.
321
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
322
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
323
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
324
- Kubernetes namespace to use when launching pod in Kubernetes.
325
- gpu : int, optional, default None
326
- Number of GPUs required for this step. A value of zero implies that
327
- the scheduled node should not have GPUs.
328
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
329
- The vendor of the GPUs to be used for this step.
330
- tolerations : List[str], default []
331
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
332
- Kubernetes tolerations to use when launching pod in Kubernetes.
333
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
334
- Kubernetes labels to use when launching pod in Kubernetes.
335
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
336
- Kubernetes annotations to use when launching pod in Kubernetes.
337
- use_tmpfs : bool, default False
338
- This enables an explicit tmpfs mount for this step.
339
- tmpfs_tempdir : bool, default True
340
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
341
- tmpfs_size : int, optional, default: None
342
- The value for the size (in MiB) of the tmpfs mount for this step.
343
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
344
- memory allocated for this step.
345
- tmpfs_path : str, optional, default /metaflow_temp
346
- Path to tmpfs mount for this step.
347
- persistent_volume_claims : Dict[str, str], optional, default None
348
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
349
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
350
- shared_memory: int, optional
351
- Shared memory size (in MiB) required for this step
352
- port: int, optional
353
- Port number to specify in the Kubernetes job object
354
- compute_pool : str, optional, default None
355
- Compute pool to be used for for this step.
356
- If not specified, any accessible compute pool within the perimeter is used.
357
- hostname_resolution_timeout: int, default 10 * 60
358
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
359
- Only applicable when @parallel is used.
360
- qos: str, default: Burstable
361
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
362
-
363
- security_context: Dict[str, Any], optional, default None
364
- Container security context. Applies to the task container. Allows the following keys:
365
- - privileged: bool, optional, default None
366
- - allow_privilege_escalation: bool, optional, default None
367
- - run_as_user: int, optional, default None
368
- - run_as_group: int, optional, default None
369
- - run_as_non_root: bool, optional, default None
267
+ sources : List[Union[str, Dict[str, Any]]], default: []
268
+ List of secret specs, defining how the secrets are to be retrieved
370
269
  """
371
270
  ...
372
271
 
@@ -413,216 +312,68 @@ def ollama(*, models: list, backend: str, force_pull: bool, cache_update_policy:
413
312
  """
414
313
  ...
415
314
 
416
- def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
315
+ @typing.overload
316
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
417
317
  """
418
- Specifies that this step should execute on DGX cloud.
318
+ Specifies the PyPI packages for the step.
319
+
320
+ Information in this decorator will augment any
321
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
322
+ you can use `@pypi_base` to set packages required by all
323
+ steps and use `@pypi` to specify step-specific overrides.
419
324
 
420
325
 
421
326
  Parameters
422
327
  ----------
423
- gpu : int
424
- Number of GPUs to use.
425
- gpu_type : str
426
- Type of Nvidia GPU to use.
427
- queue_timeout : int
428
- Time to keep the job in NVCF's queue.
429
- """
430
- ...
431
-
432
- @typing.overload
433
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
434
- """
435
- Internal decorator to support Fast bakery
436
- """
437
- ...
438
-
439
- @typing.overload
440
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
441
- ...
442
-
443
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
444
- """
445
- Internal decorator to support Fast bakery
446
- """
447
- ...
448
-
449
- @typing.overload
450
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
451
- """
452
- Specifies the resources needed when executing this step.
453
-
454
- Use `@resources` to specify the resource requirements
455
- independently of the specific compute layer (`@batch`, `@kubernetes`).
456
-
457
- You can choose the compute layer on the command line by executing e.g.
458
- ```
459
- python myflow.py run --with batch
460
- ```
461
- or
462
- ```
463
- python myflow.py run --with kubernetes
464
- ```
465
- which executes the flow on the desired system using the
466
- requirements specified in `@resources`.
467
-
468
-
469
- Parameters
470
- ----------
471
- cpu : int, default 1
472
- Number of CPUs required for this step.
473
- gpu : int, optional, default None
474
- Number of GPUs required for this step.
475
- disk : int, optional, default None
476
- Disk size (in MB) required for this step. Only applies on Kubernetes.
477
- memory : int, default 4096
478
- Memory size (in MB) required for this step.
479
- shared_memory : int, optional, default None
480
- The value for the size (in MiB) of the /dev/shm volume for this step.
481
- This parameter maps to the `--shm-size` option in Docker.
328
+ packages : Dict[str, str], default: {}
329
+ Packages to use for this step. The key is the name of the package
330
+ and the value is the version to use.
331
+ python : str, optional, default: None
332
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
333
+ that the version used will correspond to the version of the Python interpreter used to start the run.
482
334
  """
483
335
  ...
484
336
 
485
337
  @typing.overload
486
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
338
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
487
339
  ...
488
340
 
489
341
  @typing.overload
490
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
491
- ...
492
-
493
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
494
- """
495
- Specifies the resources needed when executing this step.
496
-
497
- Use `@resources` to specify the resource requirements
498
- independently of the specific compute layer (`@batch`, `@kubernetes`).
499
-
500
- You can choose the compute layer on the command line by executing e.g.
501
- ```
502
- python myflow.py run --with batch
503
- ```
504
- or
505
- ```
506
- python myflow.py run --with kubernetes
507
- ```
508
- which executes the flow on the desired system using the
509
- requirements specified in `@resources`.
510
-
511
-
512
- Parameters
513
- ----------
514
- cpu : int, default 1
515
- Number of CPUs required for this step.
516
- gpu : int, optional, default None
517
- Number of GPUs required for this step.
518
- disk : int, optional, default None
519
- Disk size (in MB) required for this step. Only applies on Kubernetes.
520
- memory : int, default 4096
521
- Memory size (in MB) required for this step.
522
- shared_memory : int, optional, default None
523
- The value for the size (in MiB) of the /dev/shm volume for this step.
524
- This parameter maps to the `--shm-size` option in Docker.
525
- """
526
- ...
527
-
528
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
529
- """
530
- Decorator that helps cache, version and store models/datasets from huggingface hub.
531
-
532
-
533
- Parameters
534
- ----------
535
- temp_dir_root : str, optional
536
- The root directory that will hold the temporary directory where objects will be downloaded.
537
-
538
- load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
539
- The list of repos (models/datasets) to load.
540
-
541
- Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
542
-
543
- - If repo (model/dataset) is not found in the datastore:
544
- - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
545
- - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
546
- - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
547
-
548
- - If repo is found in the datastore:
549
- - Loads it directly from datastore to local path (can be temporary directory or specified path)
550
- """
342
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
551
343
  ...
552
344
 
553
- @typing.overload
554
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
345
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
555
346
  """
556
- Enables loading / saving of models within a step.
347
+ Specifies the PyPI packages for the step.
557
348
 
349
+ Information in this decorator will augment any
350
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
351
+ you can use `@pypi_base` to set packages required by all
352
+ steps and use `@pypi` to specify step-specific overrides.
558
353
 
559
354
 
560
355
  Parameters
561
356
  ----------
562
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
563
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
564
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
565
- - `current.checkpoint`
566
- - `current.model`
567
- - `current.huggingface_hub`
568
-
569
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
570
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
571
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
572
-
573
- temp_dir_root : str, default: None
574
- The root directory under which `current.model.loaded` will store loaded models
357
+ packages : Dict[str, str], default: {}
358
+ Packages to use for this step. The key is the name of the package
359
+ and the value is the version to use.
360
+ python : str, optional, default: None
361
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
362
+ that the version used will correspond to the version of the Python interpreter used to start the run.
575
363
  """
576
364
  ...
577
365
 
578
- @typing.overload
579
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
580
- ...
581
-
582
- @typing.overload
583
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
584
- ...
585
-
586
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
366
+ def nvct(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
587
367
  """
588
- Enables loading / saving of models within a step.
589
-
368
+ Specifies that this step should execute on DGX cloud.
590
369
 
591
370
 
592
371
  Parameters
593
372
  ----------
594
- load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
595
- Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
596
- These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
597
- - `current.checkpoint`
598
- - `current.model`
599
- - `current.huggingface_hub`
600
-
601
- If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
602
- the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
603
- If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
604
-
605
- temp_dir_root : str, default: None
606
- The root directory under which `current.model.loaded` will store loaded models
607
- """
608
- ...
609
-
610
- @typing.overload
611
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
612
- """
613
- Decorator prototype for all step decorators. This function gets specialized
614
- and imported for all decorators types by _import_plugin_decorators().
615
- """
616
- ...
617
-
618
- @typing.overload
619
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
620
- ...
621
-
622
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
623
- """
624
- Decorator prototype for all step decorators. This function gets specialized
625
- and imported for all decorators types by _import_plugin_decorators().
373
+ gpu : int
374
+ Number of GPUs to use.
375
+ gpu_type : str
376
+ Type of Nvidia GPU to use.
626
377
  """
627
378
  ...
628
379
 
@@ -686,54 +437,182 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
686
437
  ...
687
438
 
688
439
  @typing.overload
689
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
440
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
690
441
  """
691
- Specifies environment variables to be set prior to the execution of a step.
442
+ Enables loading / saving of models within a step.
443
+
692
444
 
693
445
 
694
446
  Parameters
695
447
  ----------
696
- vars : Dict[str, str], default {}
697
- Dictionary of environment variables to set.
448
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
449
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
450
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
451
+ - `current.checkpoint`
452
+ - `current.model`
453
+ - `current.huggingface_hub`
454
+
455
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
456
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
457
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
458
+
459
+ temp_dir_root : str, default: None
460
+ The root directory under which `current.model.loaded` will store loaded models
698
461
  """
699
462
  ...
700
463
 
701
464
  @typing.overload
702
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
465
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
703
466
  ...
704
467
 
705
468
  @typing.overload
706
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
469
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
707
470
  ...
708
471
 
709
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
472
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
710
473
  """
711
- Specifies environment variables to be set prior to the execution of a step.
474
+ Enables loading / saving of models within a step.
475
+
712
476
 
713
477
 
714
478
  Parameters
715
479
  ----------
716
- vars : Dict[str, str], default {}
717
- Dictionary of environment variables to set.
480
+ load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
481
+ Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
482
+ These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
483
+ - `current.checkpoint`
484
+ - `current.model`
485
+ - `current.huggingface_hub`
486
+
487
+ If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
488
+ the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
489
+ If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
490
+
491
+ temp_dir_root : str, default: None
492
+ The root directory under which `current.model.loaded` will store loaded models
718
493
  """
719
494
  ...
720
495
 
721
- @typing.overload
722
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
496
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
723
497
  """
724
- Specifies the Conda environment for the step.
725
-
726
- Information in this decorator will augment any
727
- attributes set in the `@conda_base` flow-level decorator. Hence,
728
- you can use `@conda_base` to set packages required by all
729
- steps and use `@conda` to specify step-specific overrides.
498
+ Specifies that this step should execute on Kubernetes.
730
499
 
731
500
 
732
501
  Parameters
733
502
  ----------
734
- packages : Dict[str, str], default {}
735
- Packages to use for this step. The key is the name of the package
736
- and the value is the version to use.
503
+ cpu : int, default 1
504
+ Number of CPUs required for this step. If `@resources` is
505
+ also present, the maximum value from all decorators is used.
506
+ memory : int, default 4096
507
+ Memory size (in MB) required for this step. If
508
+ `@resources` is also present, the maximum value from all decorators is
509
+ used.
510
+ disk : int, default 10240
511
+ Disk size (in MB) required for this step. If
512
+ `@resources` is also present, the maximum value from all decorators is
513
+ used.
514
+ image : str, optional, default None
515
+ Docker image to use when launching on Kubernetes. If not specified, and
516
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
517
+ not, a default Docker image mapping to the current version of Python is used.
518
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
519
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
520
+ image_pull_secrets: List[str], default []
521
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
522
+ Kubernetes image pull secrets to use when pulling container images
523
+ in Kubernetes.
524
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
525
+ Kubernetes service account to use when launching pod in Kubernetes.
526
+ secrets : List[str], optional, default None
527
+ Kubernetes secrets to use when launching pod in Kubernetes. These
528
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
529
+ in Metaflow configuration.
530
+ node_selector: Union[Dict[str,str], str], optional, default None
531
+ Kubernetes node selector(s) to apply to the pod running the task.
532
+ Can be passed in as a comma separated string of values e.g.
533
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
534
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
535
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
536
+ Kubernetes namespace to use when launching pod in Kubernetes.
537
+ gpu : int, optional, default None
538
+ Number of GPUs required for this step. A value of zero implies that
539
+ the scheduled node should not have GPUs.
540
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
541
+ The vendor of the GPUs to be used for this step.
542
+ tolerations : List[str], default []
543
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
544
+ Kubernetes tolerations to use when launching pod in Kubernetes.
545
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
546
+ Kubernetes labels to use when launching pod in Kubernetes.
547
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
548
+ Kubernetes annotations to use when launching pod in Kubernetes.
549
+ use_tmpfs : bool, default False
550
+ This enables an explicit tmpfs mount for this step.
551
+ tmpfs_tempdir : bool, default True
552
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
553
+ tmpfs_size : int, optional, default: None
554
+ The value for the size (in MiB) of the tmpfs mount for this step.
555
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
556
+ memory allocated for this step.
557
+ tmpfs_path : str, optional, default /metaflow_temp
558
+ Path to tmpfs mount for this step.
559
+ persistent_volume_claims : Dict[str, str], optional, default None
560
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
561
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
562
+ shared_memory: int, optional
563
+ Shared memory size (in MiB) required for this step
564
+ port: int, optional
565
+ Port number to specify in the Kubernetes job object
566
+ compute_pool : str, optional, default None
567
+ Compute pool to be used for for this step.
568
+ If not specified, any accessible compute pool within the perimeter is used.
569
+ hostname_resolution_timeout: int, default 10 * 60
570
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
571
+ Only applicable when @parallel is used.
572
+ qos: str, default: Burstable
573
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
574
+
575
+ security_context: Dict[str, Any], optional, default None
576
+ Container security context. Applies to the task container. Allows the following keys:
577
+ - privileged: bool, optional, default None
578
+ - allow_privilege_escalation: bool, optional, default None
579
+ - run_as_user: int, optional, default None
580
+ - run_as_group: int, optional, default None
581
+ - run_as_non_root: bool, optional, default None
582
+ """
583
+ ...
584
+
585
+ def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
586
+ """
587
+ Specifies that this step is used to deploy an instance of the app.
588
+ Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
589
+
590
+
591
+ Parameters
592
+ ----------
593
+ app_port : int
594
+ Number of GPUs to use.
595
+ app_name : str
596
+ Name of the app to deploy.
597
+ """
598
+ ...
599
+
600
+ @typing.overload
601
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
602
+ """
603
+ Specifies the Conda environment for the step.
604
+
605
+ Information in this decorator will augment any
606
+ attributes set in the `@conda_base` flow-level decorator. Hence,
607
+ you can use `@conda_base` to set packages required by all
608
+ steps and use `@conda` to specify step-specific overrides.
609
+
610
+
611
+ Parameters
612
+ ----------
613
+ packages : Dict[str, str], default {}
614
+ Packages to use for this step. The key is the name of the package
615
+ and the value is the version to use.
737
616
  libraries : Dict[str, str], default {}
738
617
  Supported for backward compatibility. When used with packages, packages will take precedence.
739
618
  python : str, optional, default None
@@ -778,154 +657,174 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
778
657
  ...
779
658
 
780
659
  @typing.overload
781
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
660
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
782
661
  """
783
- Specifies that the step will success under all circumstances.
784
-
785
- The decorator will create an optional artifact, specified by `var`, which
786
- contains the exception raised. You can use it to detect the presence
787
- of errors, indicating that all happy-path artifacts produced by the step
788
- are missing.
789
-
790
-
791
- Parameters
792
- ----------
793
- var : str, optional, default None
794
- Name of the artifact in which to store the caught exception.
795
- If not specified, the exception is not stored.
796
- print_exception : bool, default True
797
- Determines whether or not the exception is printed to
798
- stdout when caught.
662
+ Decorator prototype for all step decorators. This function gets specialized
663
+ and imported for all decorators types by _import_plugin_decorators().
799
664
  """
800
665
  ...
801
666
 
802
667
  @typing.overload
803
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
668
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
804
669
  ...
805
670
 
806
- @typing.overload
807
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
671
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
672
+ """
673
+ Decorator prototype for all step decorators. This function gets specialized
674
+ and imported for all decorators types by _import_plugin_decorators().
675
+ """
808
676
  ...
809
677
 
810
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
678
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
811
679
  """
812
- Specifies that the step will success under all circumstances.
813
-
814
- The decorator will create an optional artifact, specified by `var`, which
815
- contains the exception raised. You can use it to detect the presence
816
- of errors, indicating that all happy-path artifacts produced by the step
817
- are missing.
680
+ Specifies that this step should execute on DGX cloud.
818
681
 
819
682
 
820
683
  Parameters
821
684
  ----------
822
- var : str, optional, default None
823
- Name of the artifact in which to store the caught exception.
824
- If not specified, the exception is not stored.
825
- print_exception : bool, default True
826
- Determines whether or not the exception is printed to
827
- stdout when caught.
685
+ gpu : int
686
+ Number of GPUs to use.
687
+ gpu_type : str
688
+ Type of Nvidia GPU to use.
689
+ queue_timeout : int
690
+ Time to keep the job in NVCF's queue.
828
691
  """
829
692
  ...
830
693
 
831
694
  @typing.overload
832
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
695
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
833
696
  """
834
- Specifies secrets to be retrieved and injected as environment variables prior to
835
- the execution of a step.
697
+ Enables checkpointing for a step.
698
+
836
699
 
837
700
 
838
701
  Parameters
839
702
  ----------
840
- sources : List[Union[str, Dict[str, Any]]], default: []
841
- List of secret specs, defining how the secrets are to be retrieved
703
+ load_policy : str, default: "fresh"
704
+ The policy for loading the checkpoint. The following policies are supported:
705
+ - "eager": Loads the the latest available checkpoint within the namespace.
706
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
707
+ will be loaded at the start of the task.
708
+ - "none": Do not load any checkpoint
709
+ - "fresh": Loads the lastest checkpoint created within the running Task.
710
+ This mode helps loading checkpoints across various retry attempts of the same task.
711
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
712
+ created within the task will be loaded when the task is retries execution on failure.
713
+
714
+ temp_dir_root : str, default: None
715
+ The root directory under which `current.checkpoint.directory` will be created.
842
716
  """
843
717
  ...
844
718
 
845
719
  @typing.overload
846
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
720
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
847
721
  ...
848
722
 
849
723
  @typing.overload
850
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
724
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
851
725
  ...
852
726
 
853
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
727
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
854
728
  """
855
- Specifies secrets to be retrieved and injected as environment variables prior to
856
- the execution of a step.
729
+ Enables checkpointing for a step.
730
+
857
731
 
858
732
 
859
733
  Parameters
860
734
  ----------
861
- sources : List[Union[str, Dict[str, Any]]], default: []
862
- List of secret specs, defining how the secrets are to be retrieved
735
+ load_policy : str, default: "fresh"
736
+ The policy for loading the checkpoint. The following policies are supported:
737
+ - "eager": Loads the the latest available checkpoint within the namespace.
738
+ With this mode, the latest checkpoint written by any previous task (can be even a different run) of the step
739
+ will be loaded at the start of the task.
740
+ - "none": Do not load any checkpoint
741
+ - "fresh": Loads the lastest checkpoint created within the running Task.
742
+ This mode helps loading checkpoints across various retry attempts of the same task.
743
+ With this mode, no checkpoint will be loaded at the start of a task but any checkpoints
744
+ created within the task will be loaded when the task is retries execution on failure.
745
+
746
+ temp_dir_root : str, default: None
747
+ The root directory under which `current.checkpoint.directory` will be created.
863
748
  """
864
749
  ...
865
750
 
866
751
  @typing.overload
867
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
752
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
868
753
  """
869
- Specifies the PyPI packages for the step.
754
+ Specifies the number of times the task corresponding
755
+ to a step needs to be retried.
870
756
 
871
- Information in this decorator will augment any
872
- attributes set in the `@pyi_base` flow-level decorator. Hence,
873
- you can use `@pypi_base` to set packages required by all
874
- steps and use `@pypi` to specify step-specific overrides.
757
+ This decorator is useful for handling transient errors, such as networking issues.
758
+ If your task contains operations that can't be retried safely, e.g. database updates,
759
+ it is advisable to annotate it with `@retry(times=0)`.
760
+
761
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
762
+ decorator will execute a no-op task after all retries have been exhausted,
763
+ ensuring that the flow execution can continue.
875
764
 
876
765
 
877
766
  Parameters
878
767
  ----------
879
- packages : Dict[str, str], default: {}
880
- Packages to use for this step. The key is the name of the package
881
- and the value is the version to use.
882
- python : str, optional, default: None
883
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
884
- that the version used will correspond to the version of the Python interpreter used to start the run.
768
+ times : int, default 3
769
+ Number of times to retry this task.
770
+ minutes_between_retries : int, default 2
771
+ Number of minutes between retries.
885
772
  """
886
773
  ...
887
774
 
888
775
  @typing.overload
889
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
776
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
890
777
  ...
891
778
 
892
779
  @typing.overload
893
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
780
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
894
781
  ...
895
782
 
896
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
783
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
897
784
  """
898
- Specifies the PyPI packages for the step.
785
+ Specifies the number of times the task corresponding
786
+ to a step needs to be retried.
899
787
 
900
- Information in this decorator will augment any
901
- attributes set in the `@pyi_base` flow-level decorator. Hence,
902
- you can use `@pypi_base` to set packages required by all
903
- steps and use `@pypi` to specify step-specific overrides.
788
+ This decorator is useful for handling transient errors, such as networking issues.
789
+ If your task contains operations that can't be retried safely, e.g. database updates,
790
+ it is advisable to annotate it with `@retry(times=0)`.
791
+
792
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
793
+ decorator will execute a no-op task after all retries have been exhausted,
794
+ ensuring that the flow execution can continue.
904
795
 
905
796
 
906
797
  Parameters
907
798
  ----------
908
- packages : Dict[str, str], default: {}
909
- Packages to use for this step. The key is the name of the package
910
- and the value is the version to use.
911
- python : str, optional, default: None
912
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
913
- that the version used will correspond to the version of the Python interpreter used to start the run.
799
+ times : int, default 3
800
+ Number of times to retry this task.
801
+ minutes_between_retries : int, default 2
802
+ Number of minutes between retries.
914
803
  """
915
804
  ...
916
805
 
917
- def app_deploy(*, app_port: int, app_name: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
806
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
918
807
  """
919
- Specifies that this step is used to deploy an instance of the app.
920
- Requires that self.app_name, self.app_port, self.entrypoint and self.deployDir is set.
808
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
921
809
 
922
810
 
923
811
  Parameters
924
812
  ----------
925
- app_port : int
926
- Number of GPUs to use.
927
- app_name : str
928
- Name of the app to deploy.
813
+ temp_dir_root : str, optional
814
+ The root directory that will hold the temporary directory where objects will be downloaded.
815
+
816
+ load: Union[List[str], List[Tuple[Dict, str]], List[Tuple[str, str]], List[Dict], None]
817
+ The list of repos (models/datasets) to load.
818
+
819
+ Loaded repos can be accessed via `current.huggingface_hub.loaded`. If load is set, then the following happens:
820
+
821
+ - If repo (model/dataset) is not found in the datastore:
822
+ - Downloads the repo from Hugging Face Hub to a temporary directory (or uses specified path) for local access
823
+ - Stores it in Metaflow's datastore (s3/gcs/azure etc.) with a unique name based on repo_type/repo_id
824
+ - All HF models loaded for a `@step` will be cached separately under flow/step/namespace.
825
+
826
+ - If repo is found in the datastore:
827
+ - Loads it directly from datastore to local path (can be temporary directory or specified path)
929
828
  """
930
829
  ...
931
830
 
@@ -979,305 +878,186 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
979
878
  ...
980
879
 
981
880
  @typing.overload
982
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
881
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
983
882
  """
984
- Specifies the event(s) that this flow depends on.
883
+ Internal decorator to support Fast bakery
884
+ """
885
+ ...
886
+
887
+ @typing.overload
888
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
889
+ ...
890
+
891
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
892
+ """
893
+ Internal decorator to support Fast bakery
894
+ """
895
+ ...
896
+
897
+ @typing.overload
898
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
899
+ """
900
+ Specifies that the step will success under all circumstances.
985
901
 
986
- ```
987
- @trigger(event='foo')
988
- ```
989
- or
990
- ```
991
- @trigger(events=['foo', 'bar'])
992
- ```
902
+ The decorator will create an optional artifact, specified by `var`, which
903
+ contains the exception raised. You can use it to detect the presence
904
+ of errors, indicating that all happy-path artifacts produced by the step
905
+ are missing.
993
906
 
994
- Additionally, you can specify the parameter mappings
995
- to map event payload to Metaflow parameters for the flow.
996
- ```
997
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
998
- ```
999
- or
1000
- ```
1001
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1002
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1003
- ```
1004
907
 
1005
- 'parameters' can also be a list of strings and tuples like so:
1006
- ```
1007
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1008
- ```
1009
- This is equivalent to:
1010
- ```
1011
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1012
- ```
908
+ Parameters
909
+ ----------
910
+ var : str, optional, default None
911
+ Name of the artifact in which to store the caught exception.
912
+ If not specified, the exception is not stored.
913
+ print_exception : bool, default True
914
+ Determines whether or not the exception is printed to
915
+ stdout when caught.
916
+ """
917
+ ...
918
+
919
+ @typing.overload
920
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
921
+ ...
922
+
923
+ @typing.overload
924
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
925
+ ...
926
+
927
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
928
+ """
929
+ Specifies that the step will success under all circumstances.
930
+
931
+ The decorator will create an optional artifact, specified by `var`, which
932
+ contains the exception raised. You can use it to detect the presence
933
+ of errors, indicating that all happy-path artifacts produced by the step
934
+ are missing.
1013
935
 
1014
936
 
1015
937
  Parameters
1016
938
  ----------
1017
- event : Union[str, Dict[str, Any]], optional, default None
1018
- Event dependency for this flow.
1019
- events : List[Union[str, Dict[str, Any]]], default []
1020
- Events dependency for this flow.
1021
- options : Dict[str, Any], default {}
1022
- Backend-specific configuration for tuning eventing behavior.
939
+ var : str, optional, default None
940
+ Name of the artifact in which to store the caught exception.
941
+ If not specified, the exception is not stored.
942
+ print_exception : bool, default True
943
+ Determines whether or not the exception is printed to
944
+ stdout when caught.
1023
945
  """
1024
946
  ...
1025
947
 
1026
948
  @typing.overload
1027
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
949
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
950
+ """
951
+ Specifies environment variables to be set prior to the execution of a step.
952
+
953
+
954
+ Parameters
955
+ ----------
956
+ vars : Dict[str, str], default {}
957
+ Dictionary of environment variables to set.
958
+ """
1028
959
  ...
1029
960
 
1030
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
961
+ @typing.overload
962
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
963
+ ...
964
+
965
+ @typing.overload
966
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
967
+ ...
968
+
969
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1031
970
  """
1032
- Specifies the event(s) that this flow depends on.
971
+ Specifies environment variables to be set prior to the execution of a step.
1033
972
 
1034
- ```
1035
- @trigger(event='foo')
1036
- ```
1037
- or
1038
- ```
1039
- @trigger(events=['foo', 'bar'])
1040
- ```
1041
973
 
1042
- Additionally, you can specify the parameter mappings
1043
- to map event payload to Metaflow parameters for the flow.
1044
- ```
1045
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1046
- ```
1047
- or
1048
- ```
1049
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1050
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1051
- ```
974
+ Parameters
975
+ ----------
976
+ vars : Dict[str, str], default {}
977
+ Dictionary of environment variables to set.
978
+ """
979
+ ...
980
+
981
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
982
+ """
983
+ Specifies what flows belong to the same project.
1052
984
 
1053
- 'parameters' can also be a list of strings and tuples like so:
1054
- ```
1055
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1056
- ```
1057
- This is equivalent to:
1058
- ```
1059
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1060
- ```
985
+ A project-specific namespace is created for all flows that
986
+ use the same `@project(name)`.
1061
987
 
1062
988
 
1063
989
  Parameters
1064
990
  ----------
1065
- event : Union[str, Dict[str, Any]], optional, default None
1066
- Event dependency for this flow.
1067
- events : List[Union[str, Dict[str, Any]]], default []
1068
- Events dependency for this flow.
1069
- options : Dict[str, Any], default {}
1070
- Backend-specific configuration for tuning eventing behavior.
991
+ name : str
992
+ Project name. Make sure that the name is unique amongst all
993
+ projects that use the same production scheduler. The name may
994
+ contain only lowercase alphanumeric characters and underscores.
995
+
996
+ branch : Optional[str], default None
997
+ The branch to use. If not specified, the branch is set to
998
+ `user.<username>` unless `production` is set to `True`. This can
999
+ also be set on the command line using `--branch` as a top-level option.
1000
+ It is an error to specify `branch` in the decorator and on the command line.
1001
+
1002
+ production : bool, default False
1003
+ Whether or not the branch is the production branch. This can also be set on the
1004
+ command line using `--production` as a top-level option. It is an error to specify
1005
+ `production` in the decorator and on the command line.
1006
+ The project branch name will be:
1007
+ - if `branch` is specified:
1008
+ - if `production` is True: `prod.<branch>`
1009
+ - if `production` is False: `test.<branch>`
1010
+ - if `branch` is not specified:
1011
+ - if `production` is True: `prod`
1012
+ - if `production` is False: `user.<username>`
1071
1013
  """
1072
1014
  ...
1073
1015
 
1074
1016
  @typing.overload
1075
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1017
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1076
1018
  """
1077
- Specifies the Conda environment for all steps of the flow.
1078
-
1079
- Use `@conda_base` to set common libraries required by all
1080
- steps and use `@conda` to specify step-specific additions.
1019
+ Specifies the PyPI packages for all steps of the flow.
1081
1020
 
1021
+ Use `@pypi_base` to set common packages required by all
1022
+ steps and use `@pypi` to specify step-specific overrides.
1082
1023
 
1083
1024
  Parameters
1084
1025
  ----------
1085
- packages : Dict[str, str], default {}
1026
+ packages : Dict[str, str], default: {}
1086
1027
  Packages to use for this flow. The key is the name of the package
1087
1028
  and the value is the version to use.
1088
- libraries : Dict[str, str], default {}
1089
- Supported for backward compatibility. When used with packages, packages will take precedence.
1090
- python : str, optional, default None
1029
+ python : str, optional, default: None
1091
1030
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1092
1031
  that the version used will correspond to the version of the Python interpreter used to start the run.
1093
- disabled : bool, default False
1094
- If set to True, disables Conda.
1095
1032
  """
1096
1033
  ...
1097
1034
 
1098
1035
  @typing.overload
1099
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1036
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1100
1037
  ...
1101
1038
 
1102
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1039
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1103
1040
  """
1104
- Specifies the Conda environment for all steps of the flow.
1105
-
1106
- Use `@conda_base` to set common libraries required by all
1107
- steps and use `@conda` to specify step-specific additions.
1041
+ Specifies the PyPI packages for all steps of the flow.
1108
1042
 
1043
+ Use `@pypi_base` to set common packages required by all
1044
+ steps and use `@pypi` to specify step-specific overrides.
1109
1045
 
1110
1046
  Parameters
1111
1047
  ----------
1112
- packages : Dict[str, str], default {}
1048
+ packages : Dict[str, str], default: {}
1113
1049
  Packages to use for this flow. The key is the name of the package
1114
1050
  and the value is the version to use.
1115
- libraries : Dict[str, str], default {}
1116
- Supported for backward compatibility. When used with packages, packages will take precedence.
1117
- python : str, optional, default None
1051
+ python : str, optional, default: None
1118
1052
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1119
1053
  that the version used will correspond to the version of the Python interpreter used to start the run.
1120
- disabled : bool, default False
1121
- If set to True, disables Conda.
1122
1054
  """
1123
1055
  ...
1124
1056
 
1125
- @typing.overload
1126
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1057
+ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1127
1058
  """
1128
- Specifies the flow(s) that this flow depends on.
1129
-
1130
- ```
1131
- @trigger_on_finish(flow='FooFlow')
1132
- ```
1133
- or
1134
- ```
1135
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1136
- ```
1137
- This decorator respects the @project decorator and triggers the flow
1138
- when upstream runs within the same namespace complete successfully
1139
-
1140
- Additionally, you can specify project aware upstream flow dependencies
1141
- by specifying the fully qualified project_flow_name.
1142
- ```
1143
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1144
- ```
1145
- or
1146
- ```
1147
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1148
- ```
1149
-
1150
- You can also specify just the project or project branch (other values will be
1151
- inferred from the current project or project branch):
1152
- ```
1153
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1154
- ```
1155
-
1156
- Note that `branch` is typically one of:
1157
- - `prod`
1158
- - `user.bob`
1159
- - `test.my_experiment`
1160
- - `prod.staging`
1161
-
1162
-
1163
- Parameters
1164
- ----------
1165
- flow : Union[str, Dict[str, str]], optional, default None
1166
- Upstream flow dependency for this flow.
1167
- flows : List[Union[str, Dict[str, str]]], default []
1168
- Upstream flow dependencies for this flow.
1169
- options : Dict[str, Any], default {}
1170
- Backend-specific configuration for tuning eventing behavior.
1171
- """
1172
- ...
1173
-
1174
- @typing.overload
1175
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1176
- ...
1177
-
1178
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1179
- """
1180
- Specifies the flow(s) that this flow depends on.
1181
-
1182
- ```
1183
- @trigger_on_finish(flow='FooFlow')
1184
- ```
1185
- or
1186
- ```
1187
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1188
- ```
1189
- This decorator respects the @project decorator and triggers the flow
1190
- when upstream runs within the same namespace complete successfully
1191
-
1192
- Additionally, you can specify project aware upstream flow dependencies
1193
- by specifying the fully qualified project_flow_name.
1194
- ```
1195
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1196
- ```
1197
- or
1198
- ```
1199
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1200
- ```
1201
-
1202
- You can also specify just the project or project branch (other values will be
1203
- inferred from the current project or project branch):
1204
- ```
1205
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1206
- ```
1207
-
1208
- Note that `branch` is typically one of:
1209
- - `prod`
1210
- - `user.bob`
1211
- - `test.my_experiment`
1212
- - `prod.staging`
1213
-
1214
-
1215
- Parameters
1216
- ----------
1217
- flow : Union[str, Dict[str, str]], optional, default None
1218
- Upstream flow dependency for this flow.
1219
- flows : List[Union[str, Dict[str, str]]], default []
1220
- Upstream flow dependencies for this flow.
1221
- options : Dict[str, Any], default {}
1222
- Backend-specific configuration for tuning eventing behavior.
1223
- """
1224
- ...
1225
-
1226
- @typing.overload
1227
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1228
- """
1229
- Specifies the times when the flow should be run when running on a
1230
- production scheduler.
1231
-
1232
-
1233
- Parameters
1234
- ----------
1235
- hourly : bool, default False
1236
- Run the workflow hourly.
1237
- daily : bool, default True
1238
- Run the workflow daily.
1239
- weekly : bool, default False
1240
- Run the workflow weekly.
1241
- cron : str, optional, default None
1242
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1243
- specified by this expression.
1244
- timezone : str, optional, default None
1245
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1246
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1247
- """
1248
- ...
1249
-
1250
- @typing.overload
1251
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1252
- ...
1253
-
1254
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1255
- """
1256
- Specifies the times when the flow should be run when running on a
1257
- production scheduler.
1258
-
1259
-
1260
- Parameters
1261
- ----------
1262
- hourly : bool, default False
1263
- Run the workflow hourly.
1264
- daily : bool, default True
1265
- Run the workflow daily.
1266
- weekly : bool, default False
1267
- Run the workflow weekly.
1268
- cron : str, optional, default None
1269
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1270
- specified by this expression.
1271
- timezone : str, optional, default None
1272
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1273
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1274
- """
1275
- ...
1276
-
1277
- def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None):
1278
- """
1279
- Allows setting external datastores to save data for the
1280
- `@checkpoint`/`@model`/`@huggingface_hub` decorators.
1059
+ Allows setting external datastores to save data for the
1060
+ `@checkpoint`/`@model`/`@huggingface_hub` decorators.
1281
1061
 
1282
1062
  This decorator is useful when users wish to save data to a different datastore
1283
1063
  than what is configured in Metaflow. This can be for variety of reasons:
@@ -1388,86 +1168,13 @@ def with_artifact_store(f: typing.Optional[typing.Type[FlowSpecDerived]] = None)
1388
1168
  """
1389
1169
  ...
1390
1170
 
1391
- @typing.overload
1392
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1393
- """
1394
- Specifies the PyPI packages for all steps of the flow.
1395
-
1396
- Use `@pypi_base` to set common packages required by all
1397
- steps and use `@pypi` to specify step-specific overrides.
1398
-
1399
- Parameters
1400
- ----------
1401
- packages : Dict[str, str], default: {}
1402
- Packages to use for this flow. The key is the name of the package
1403
- and the value is the version to use.
1404
- python : str, optional, default: None
1405
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1406
- that the version used will correspond to the version of the Python interpreter used to start the run.
1407
- """
1408
- ...
1409
-
1410
- @typing.overload
1411
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1412
- ...
1413
-
1414
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1415
- """
1416
- Specifies the PyPI packages for all steps of the flow.
1417
-
1418
- Use `@pypi_base` to set common packages required by all
1419
- steps and use `@pypi` to specify step-specific overrides.
1420
-
1421
- Parameters
1422
- ----------
1423
- packages : Dict[str, str], default: {}
1424
- Packages to use for this flow. The key is the name of the package
1425
- and the value is the version to use.
1426
- python : str, optional, default: None
1427
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1428
- that the version used will correspond to the version of the Python interpreter used to start the run.
1429
- """
1430
- ...
1431
-
1432
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1433
- """
1434
- Specifies what flows belong to the same project.
1435
-
1436
- A project-specific namespace is created for all flows that
1437
- use the same `@project(name)`.
1438
-
1439
-
1440
- Parameters
1441
- ----------
1442
- name : str
1443
- Project name. Make sure that the name is unique amongst all
1444
- projects that use the same production scheduler. The name may
1445
- contain only lowercase alphanumeric characters and underscores.
1446
-
1447
- branch : Optional[str], default None
1448
- The branch to use. If not specified, the branch is set to
1449
- `user.<username>` unless `production` is set to `True`. This can
1450
- also be set on the command line using `--branch` as a top-level option.
1451
- It is an error to specify `branch` in the decorator and on the command line.
1452
-
1453
- production : bool, default False
1454
- Whether or not the branch is the production branch. This can also be set on the
1455
- command line using `--production` as a top-level option. It is an error to specify
1456
- `production` in the decorator and on the command line.
1457
- The project branch name will be:
1458
- - if `branch` is specified:
1459
- - if `production` is True: `prod.<branch>`
1460
- - if `production` is False: `test.<branch>`
1461
- - if `branch` is not specified:
1462
- - if `production` is True: `prod`
1463
- - if `production` is False: `user.<username>`
1464
- """
1465
- ...
1466
-
1467
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1171
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1468
1172
  """
1469
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1470
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1173
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1174
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1175
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1176
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1177
+ starts only after all sensors finish.
1471
1178
 
1472
1179
 
1473
1180
  Parameters
@@ -1489,31 +1196,177 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1489
1196
  Name of the sensor on Airflow
1490
1197
  description : str
1491
1198
  Description of sensor in the Airflow UI
1492
- external_dag_id : str
1493
- The dag_id that contains the task you want to wait for.
1494
- external_task_ids : List[str]
1495
- The list of task_ids that you want to wait for.
1496
- If None (default value) the sensor waits for the DAG. (Default: None)
1497
- allowed_states : List[str]
1498
- Iterable of allowed states, (Default: ['success'])
1499
- failed_states : List[str]
1500
- Iterable of failed or dis-allowed states. (Default: None)
1501
- execution_delta : datetime.timedelta
1502
- time difference with the previous execution to look at,
1503
- the default is the same logical date as the current task or DAG. (Default: None)
1504
- check_existence: bool
1505
- Set to True to check if the external task exists or check if
1506
- the DAG to wait for exists. (Default: True)
1507
- """
1508
- ...
1509
-
1510
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1511
- """
1512
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1513
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1514
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1515
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1516
- starts only after all sensors finish.
1199
+ bucket_key : Union[str, List[str]]
1200
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1201
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1202
+ bucket_name : str
1203
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1204
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1205
+ wildcard_match : bool
1206
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1207
+ aws_conn_id : str
1208
+ a reference to the s3 connection on Airflow. (Default: None)
1209
+ verify : bool
1210
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1211
+ """
1212
+ ...
1213
+
1214
+ @typing.overload
1215
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1216
+ """
1217
+ Specifies the times when the flow should be run when running on a
1218
+ production scheduler.
1219
+
1220
+
1221
+ Parameters
1222
+ ----------
1223
+ hourly : bool, default False
1224
+ Run the workflow hourly.
1225
+ daily : bool, default True
1226
+ Run the workflow daily.
1227
+ weekly : bool, default False
1228
+ Run the workflow weekly.
1229
+ cron : str, optional, default None
1230
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1231
+ specified by this expression.
1232
+ timezone : str, optional, default None
1233
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1234
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1235
+ """
1236
+ ...
1237
+
1238
+ @typing.overload
1239
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1240
+ ...
1241
+
1242
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1243
+ """
1244
+ Specifies the times when the flow should be run when running on a
1245
+ production scheduler.
1246
+
1247
+
1248
+ Parameters
1249
+ ----------
1250
+ hourly : bool, default False
1251
+ Run the workflow hourly.
1252
+ daily : bool, default True
1253
+ Run the workflow daily.
1254
+ weekly : bool, default False
1255
+ Run the workflow weekly.
1256
+ cron : str, optional, default None
1257
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1258
+ specified by this expression.
1259
+ timezone : str, optional, default None
1260
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1261
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1262
+ """
1263
+ ...
1264
+
1265
+ @typing.overload
1266
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1267
+ """
1268
+ Specifies the flow(s) that this flow depends on.
1269
+
1270
+ ```
1271
+ @trigger_on_finish(flow='FooFlow')
1272
+ ```
1273
+ or
1274
+ ```
1275
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1276
+ ```
1277
+ This decorator respects the @project decorator and triggers the flow
1278
+ when upstream runs within the same namespace complete successfully
1279
+
1280
+ Additionally, you can specify project aware upstream flow dependencies
1281
+ by specifying the fully qualified project_flow_name.
1282
+ ```
1283
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1284
+ ```
1285
+ or
1286
+ ```
1287
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1288
+ ```
1289
+
1290
+ You can also specify just the project or project branch (other values will be
1291
+ inferred from the current project or project branch):
1292
+ ```
1293
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1294
+ ```
1295
+
1296
+ Note that `branch` is typically one of:
1297
+ - `prod`
1298
+ - `user.bob`
1299
+ - `test.my_experiment`
1300
+ - `prod.staging`
1301
+
1302
+
1303
+ Parameters
1304
+ ----------
1305
+ flow : Union[str, Dict[str, str]], optional, default None
1306
+ Upstream flow dependency for this flow.
1307
+ flows : List[Union[str, Dict[str, str]]], default []
1308
+ Upstream flow dependencies for this flow.
1309
+ options : Dict[str, Any], default {}
1310
+ Backend-specific configuration for tuning eventing behavior.
1311
+ """
1312
+ ...
1313
+
1314
+ @typing.overload
1315
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1316
+ ...
1317
+
1318
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1319
+ """
1320
+ Specifies the flow(s) that this flow depends on.
1321
+
1322
+ ```
1323
+ @trigger_on_finish(flow='FooFlow')
1324
+ ```
1325
+ or
1326
+ ```
1327
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1328
+ ```
1329
+ This decorator respects the @project decorator and triggers the flow
1330
+ when upstream runs within the same namespace complete successfully
1331
+
1332
+ Additionally, you can specify project aware upstream flow dependencies
1333
+ by specifying the fully qualified project_flow_name.
1334
+ ```
1335
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1336
+ ```
1337
+ or
1338
+ ```
1339
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1340
+ ```
1341
+
1342
+ You can also specify just the project or project branch (other values will be
1343
+ inferred from the current project or project branch):
1344
+ ```
1345
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1346
+ ```
1347
+
1348
+ Note that `branch` is typically one of:
1349
+ - `prod`
1350
+ - `user.bob`
1351
+ - `test.my_experiment`
1352
+ - `prod.staging`
1353
+
1354
+
1355
+ Parameters
1356
+ ----------
1357
+ flow : Union[str, Dict[str, str]], optional, default None
1358
+ Upstream flow dependency for this flow.
1359
+ flows : List[Union[str, Dict[str, str]]], default []
1360
+ Upstream flow dependencies for this flow.
1361
+ options : Dict[str, Any], default {}
1362
+ Backend-specific configuration for tuning eventing behavior.
1363
+ """
1364
+ ...
1365
+
1366
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1367
+ """
1368
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1369
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1517
1370
 
1518
1371
 
1519
1372
  Parameters
@@ -1535,18 +1388,165 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1535
1388
  Name of the sensor on Airflow
1536
1389
  description : str
1537
1390
  Description of sensor in the Airflow UI
1538
- bucket_key : Union[str, List[str]]
1539
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1540
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1541
- bucket_name : str
1542
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1543
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1544
- wildcard_match : bool
1545
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1546
- aws_conn_id : str
1547
- a reference to the s3 connection on Airflow. (Default: None)
1548
- verify : bool
1549
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1391
+ external_dag_id : str
1392
+ The dag_id that contains the task you want to wait for.
1393
+ external_task_ids : List[str]
1394
+ The list of task_ids that you want to wait for.
1395
+ If None (default value) the sensor waits for the DAG. (Default: None)
1396
+ allowed_states : List[str]
1397
+ Iterable of allowed states, (Default: ['success'])
1398
+ failed_states : List[str]
1399
+ Iterable of failed or dis-allowed states. (Default: None)
1400
+ execution_delta : datetime.timedelta
1401
+ time difference with the previous execution to look at,
1402
+ the default is the same logical date as the current task or DAG. (Default: None)
1403
+ check_existence: bool
1404
+ Set to True to check if the external task exists or check if
1405
+ the DAG to wait for exists. (Default: True)
1406
+ """
1407
+ ...
1408
+
1409
+ @typing.overload
1410
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1411
+ """
1412
+ Specifies the Conda environment for all steps of the flow.
1413
+
1414
+ Use `@conda_base` to set common libraries required by all
1415
+ steps and use `@conda` to specify step-specific additions.
1416
+
1417
+
1418
+ Parameters
1419
+ ----------
1420
+ packages : Dict[str, str], default {}
1421
+ Packages to use for this flow. The key is the name of the package
1422
+ and the value is the version to use.
1423
+ libraries : Dict[str, str], default {}
1424
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1425
+ python : str, optional, default None
1426
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1427
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1428
+ disabled : bool, default False
1429
+ If set to True, disables Conda.
1430
+ """
1431
+ ...
1432
+
1433
+ @typing.overload
1434
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1435
+ ...
1436
+
1437
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1438
+ """
1439
+ Specifies the Conda environment for all steps of the flow.
1440
+
1441
+ Use `@conda_base` to set common libraries required by all
1442
+ steps and use `@conda` to specify step-specific additions.
1443
+
1444
+
1445
+ Parameters
1446
+ ----------
1447
+ packages : Dict[str, str], default {}
1448
+ Packages to use for this flow. The key is the name of the package
1449
+ and the value is the version to use.
1450
+ libraries : Dict[str, str], default {}
1451
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1452
+ python : str, optional, default None
1453
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1454
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1455
+ disabled : bool, default False
1456
+ If set to True, disables Conda.
1457
+ """
1458
+ ...
1459
+
1460
+ @typing.overload
1461
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1462
+ """
1463
+ Specifies the event(s) that this flow depends on.
1464
+
1465
+ ```
1466
+ @trigger(event='foo')
1467
+ ```
1468
+ or
1469
+ ```
1470
+ @trigger(events=['foo', 'bar'])
1471
+ ```
1472
+
1473
+ Additionally, you can specify the parameter mappings
1474
+ to map event payload to Metaflow parameters for the flow.
1475
+ ```
1476
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1477
+ ```
1478
+ or
1479
+ ```
1480
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1481
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1482
+ ```
1483
+
1484
+ 'parameters' can also be a list of strings and tuples like so:
1485
+ ```
1486
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1487
+ ```
1488
+ This is equivalent to:
1489
+ ```
1490
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1491
+ ```
1492
+
1493
+
1494
+ Parameters
1495
+ ----------
1496
+ event : Union[str, Dict[str, Any]], optional, default None
1497
+ Event dependency for this flow.
1498
+ events : List[Union[str, Dict[str, Any]]], default []
1499
+ Events dependency for this flow.
1500
+ options : Dict[str, Any], default {}
1501
+ Backend-specific configuration for tuning eventing behavior.
1502
+ """
1503
+ ...
1504
+
1505
+ @typing.overload
1506
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1507
+ ...
1508
+
1509
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1510
+ """
1511
+ Specifies the event(s) that this flow depends on.
1512
+
1513
+ ```
1514
+ @trigger(event='foo')
1515
+ ```
1516
+ or
1517
+ ```
1518
+ @trigger(events=['foo', 'bar'])
1519
+ ```
1520
+
1521
+ Additionally, you can specify the parameter mappings
1522
+ to map event payload to Metaflow parameters for the flow.
1523
+ ```
1524
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1525
+ ```
1526
+ or
1527
+ ```
1528
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1529
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1530
+ ```
1531
+
1532
+ 'parameters' can also be a list of strings and tuples like so:
1533
+ ```
1534
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1535
+ ```
1536
+ This is equivalent to:
1537
+ ```
1538
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1539
+ ```
1540
+
1541
+
1542
+ Parameters
1543
+ ----------
1544
+ event : Union[str, Dict[str, Any]], optional, default None
1545
+ Event dependency for this flow.
1546
+ events : List[Union[str, Dict[str, Any]]], default []
1547
+ Events dependency for this flow.
1548
+ options : Dict[str, Any], default {}
1549
+ Backend-specific configuration for tuning eventing behavior.
1550
1550
  """
1551
1551
  ...
1552
1552