ob-metaflow-stubs 6.0.3.134__py2.py3-none-any.whl → 6.0.3.135__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (192) hide show
  1. metaflow-stubs/__init__.pyi +301 -301
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +4 -2
  20. metaflow-stubs/metaflow_current.pyi +87 -87
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +4 -4
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +3 -3
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +5 -5
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +5 -5
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +4 -4
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +2 -2
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +5 -5
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  61. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  62. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  63. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  70. metaflow-stubs/multicore_utils.pyi +2 -2
  71. metaflow-stubs/parameters.pyi +4 -4
  72. metaflow-stubs/plugins/__init__.pyi +10 -10
  73. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  74. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  75. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  76. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  77. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  78. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  79. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  80. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  82. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  83. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  84. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  85. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +5 -5
  86. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  87. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  88. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  89. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  90. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  92. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  93. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  94. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  96. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  97. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  98. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  99. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  100. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  101. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +5 -5
  102. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  103. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  105. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  106. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  107. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  108. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  109. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  110. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  111. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  112. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  113. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  114. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  116. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  117. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  118. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  119. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  120. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  121. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  123. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  124. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  125. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  126. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  128. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  129. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  130. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  131. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  132. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  133. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  134. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  135. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  136. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  138. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  139. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  140. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  141. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  142. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  143. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  144. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  146. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  147. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  148. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  149. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  150. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  151. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  152. metaflow-stubs/plugins/perimeters.pyi +2 -2
  153. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  154. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  155. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  156. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  157. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  158. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  159. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  160. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  161. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  162. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  163. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  164. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  165. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  166. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  167. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  168. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  169. metaflow-stubs/profilers/__init__.pyi +2 -2
  170. metaflow-stubs/pylint_wrapper.pyi +2 -2
  171. metaflow-stubs/runner/__init__.pyi +2 -2
  172. metaflow-stubs/runner/deployer.pyi +30 -30
  173. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  174. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  175. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  176. metaflow-stubs/runner/nbrun.pyi +2 -2
  177. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  178. metaflow-stubs/runner/utils.pyi +4 -4
  179. metaflow-stubs/system/__init__.pyi +2 -2
  180. metaflow-stubs/system/system_logger.pyi +3 -3
  181. metaflow-stubs/system/system_monitor.pyi +2 -2
  182. metaflow-stubs/tagging_util.pyi +2 -2
  183. metaflow-stubs/tuple_util.pyi +2 -2
  184. metaflow-stubs/user_configs/__init__.pyi +2 -2
  185. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  186. metaflow-stubs/user_configs/config_options.pyi +4 -4
  187. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  188. {ob_metaflow_stubs-6.0.3.134.dist-info → ob_metaflow_stubs-6.0.3.135.dist-info}/METADATA +1 -1
  189. ob_metaflow_stubs-6.0.3.135.dist-info/RECORD +192 -0
  190. ob_metaflow_stubs-6.0.3.134.dist-info/RECORD +0 -192
  191. {ob_metaflow_stubs-6.0.3.134.dist-info → ob_metaflow_stubs-6.0.3.135.dist-info}/WHEEL +0 -0
  192. {ob_metaflow_stubs-6.0.3.134.dist-info → ob_metaflow_stubs-6.0.3.135.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.13.7.1+obcheckpoint(0.1.6);ob(v1) #
4
- # Generated on 2025-01-23T20:10:29.134629 #
3
+ # MF version: 2.13.8.1+obcheckpoint(0.1.6);ob(v1) #
4
+ # Generated on 2025-01-30T21:59:34.749405 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -35,9 +35,9 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
+ from . import events as events
38
39
  from . import tuple_util as tuple_util
39
40
  from . import cards as cards
40
- from . import events as events
41
41
  from . import runner as runner
42
42
  from . import plugins as plugins
43
43
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
@@ -147,174 +147,179 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
147
147
  ...
148
148
 
149
149
  @typing.overload
150
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
150
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
151
151
  """
152
- Decorator prototype for all step decorators. This function gets specialized
153
- and imported for all decorators types by _import_plugin_decorators().
152
+ Specifies the Conda environment for the step.
153
+
154
+ Information in this decorator will augment any
155
+ attributes set in the `@conda_base` flow-level decorator. Hence,
156
+ you can use `@conda_base` to set packages required by all
157
+ steps and use `@conda` to specify step-specific overrides.
154
158
  """
155
159
  ...
156
160
 
157
161
  @typing.overload
158
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
162
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
159
163
  ...
160
164
 
161
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
162
- """
163
- Decorator prototype for all step decorators. This function gets specialized
164
- and imported for all decorators types by _import_plugin_decorators().
165
- """
165
+ @typing.overload
166
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
166
167
  ...
167
168
 
168
- @typing.overload
169
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
169
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
170
170
  """
171
- Enables checkpointing for a step.
171
+ Specifies the Conda environment for the step.
172
+
173
+ Information in this decorator will augment any
174
+ attributes set in the `@conda_base` flow-level decorator. Hence,
175
+ you can use `@conda_base` to set packages required by all
176
+ steps and use `@conda` to specify step-specific overrides.
172
177
  """
173
178
  ...
174
179
 
175
- @typing.overload
176
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
177
- ...
178
-
179
- @typing.overload
180
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
181
- ...
182
-
183
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
180
+ def nvidia(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
184
181
  """
185
- Enables checkpointing for a step.
182
+ Specifies that this step should execute on DGX cloud.
186
183
  """
187
184
  ...
188
185
 
189
186
  @typing.overload
190
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
187
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
191
188
  """
192
- Specifies a timeout for your step.
193
-
194
- This decorator is useful if this step may hang indefinitely.
195
-
196
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
197
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
198
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
189
+ Specifies that the step will success under all circumstances.
199
190
 
200
- Note that all the values specified in parameters are added together so if you specify
201
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
191
+ The decorator will create an optional artifact, specified by `var`, which
192
+ contains the exception raised. You can use it to detect the presence
193
+ of errors, indicating that all happy-path artifacts produced by the step
194
+ are missing.
202
195
  """
203
196
  ...
204
197
 
205
198
  @typing.overload
206
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
199
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
207
200
  ...
208
201
 
209
202
  @typing.overload
210
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
203
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
211
204
  ...
212
205
 
213
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
206
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
214
207
  """
215
- Specifies a timeout for your step.
216
-
217
- This decorator is useful if this step may hang indefinitely.
218
-
219
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
220
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
221
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
208
+ Specifies that the step will success under all circumstances.
222
209
 
223
- Note that all the values specified in parameters are added together so if you specify
224
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
210
+ The decorator will create an optional artifact, specified by `var`, which
211
+ contains the exception raised. You can use it to detect the presence
212
+ of errors, indicating that all happy-path artifacts produced by the step
213
+ are missing.
225
214
  """
226
215
  ...
227
216
 
228
217
  @typing.overload
229
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
218
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
230
219
  """
231
- Creates a human-readable report, a Metaflow Card, after this step completes.
232
-
233
- Note that you may add multiple `@card` decorators in a step with different parameters.
220
+ Enables checkpointing for a step.
234
221
  """
235
222
  ...
236
223
 
237
224
  @typing.overload
238
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
225
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
239
226
  ...
240
227
 
241
228
  @typing.overload
242
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
243
- ...
244
-
245
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
246
- """
247
- Creates a human-readable report, a Metaflow Card, after this step completes.
248
-
249
- Note that you may add multiple `@card` decorators in a step with different parameters.
250
- """
229
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
251
230
  ...
252
231
 
253
- def nvidia(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
232
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
254
233
  """
255
- Specifies that this step should execute on DGX cloud.
234
+ Enables checkpointing for a step.
256
235
  """
257
236
  ...
258
237
 
259
238
  @typing.overload
260
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
239
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
261
240
  """
262
- Specifies that the step will success under all circumstances.
241
+ Specifies the resources needed when executing this step.
263
242
 
264
- The decorator will create an optional artifact, specified by `var`, which
265
- contains the exception raised. You can use it to detect the presence
266
- of errors, indicating that all happy-path artifacts produced by the step
267
- are missing.
243
+ Use `@resources` to specify the resource requirements
244
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
245
+
246
+ You can choose the compute layer on the command line by executing e.g.
247
+ ```
248
+ python myflow.py run --with batch
249
+ ```
250
+ or
251
+ ```
252
+ python myflow.py run --with kubernetes
253
+ ```
254
+ which executes the flow on the desired system using the
255
+ requirements specified in `@resources`.
268
256
  """
269
257
  ...
270
258
 
271
259
  @typing.overload
272
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
260
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
273
261
  ...
274
262
 
275
263
  @typing.overload
276
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
264
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
277
265
  ...
278
266
 
279
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
267
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
280
268
  """
281
- Specifies that the step will success under all circumstances.
269
+ Specifies the resources needed when executing this step.
282
270
 
283
- The decorator will create an optional artifact, specified by `var`, which
284
- contains the exception raised. You can use it to detect the presence
285
- of errors, indicating that all happy-path artifacts produced by the step
286
- are missing.
271
+ Use `@resources` to specify the resource requirements
272
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
273
+
274
+ You can choose the compute layer on the command line by executing e.g.
275
+ ```
276
+ python myflow.py run --with batch
277
+ ```
278
+ or
279
+ ```
280
+ python myflow.py run --with kubernetes
281
+ ```
282
+ which executes the flow on the desired system using the
283
+ requirements specified in `@resources`.
287
284
  """
288
285
  ...
289
286
 
290
287
  @typing.overload
291
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
288
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
292
289
  """
293
- Specifies the PyPI packages for the step.
290
+ Specifies a timeout for your step.
294
291
 
295
- Information in this decorator will augment any
296
- attributes set in the `@pyi_base` flow-level decorator. Hence,
297
- you can use `@pypi_base` to set packages required by all
298
- steps and use `@pypi` to specify step-specific overrides.
292
+ This decorator is useful if this step may hang indefinitely.
293
+
294
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
295
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
296
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
297
+
298
+ Note that all the values specified in parameters are added together so if you specify
299
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
299
300
  """
300
301
  ...
301
302
 
302
303
  @typing.overload
303
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
304
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
304
305
  ...
305
306
 
306
307
  @typing.overload
307
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
308
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
308
309
  ...
309
310
 
310
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
311
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
311
312
  """
312
- Specifies the PyPI packages for the step.
313
+ Specifies a timeout for your step.
313
314
 
314
- Information in this decorator will augment any
315
- attributes set in the `@pyi_base` flow-level decorator. Hence,
316
- you can use `@pypi_base` to set packages required by all
317
- steps and use `@pypi` to specify step-specific overrides.
315
+ This decorator is useful if this step may hang indefinitely.
316
+
317
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
318
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
319
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
320
+
321
+ Note that all the values specified in parameters are added together so if you specify
322
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
318
323
  """
319
324
  ...
320
325
 
@@ -357,40 +362,24 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
357
362
  """
358
363
  ...
359
364
 
360
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
361
- """
362
- Decorator that helps cache, version and store models/datasets from huggingface hub.
363
- """
364
- ...
365
-
366
365
  @typing.overload
367
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
366
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
368
367
  """
369
- Specifies the Conda environment for the step.
370
-
371
- Information in this decorator will augment any
372
- attributes set in the `@conda_base` flow-level decorator. Hence,
373
- you can use `@conda_base` to set packages required by all
374
- steps and use `@conda` to specify step-specific overrides.
368
+ Enables loading / saving of models within a step.
375
369
  """
376
370
  ...
377
371
 
378
372
  @typing.overload
379
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
373
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
380
374
  ...
381
375
 
382
376
  @typing.overload
383
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
377
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
384
378
  ...
385
379
 
386
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
380
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
387
381
  """
388
- Specifies the Conda environment for the step.
389
-
390
- Information in this decorator will augment any
391
- attributes set in the `@conda_base` flow-level decorator. Hence,
392
- you can use `@conda_base` to set packages required by all
393
- steps and use `@conda` to specify step-specific overrides.
382
+ Enables loading / saving of models within a step.
394
383
  """
395
384
  ...
396
385
 
@@ -412,248 +401,176 @@ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepF
412
401
  ...
413
402
 
414
403
  @typing.overload
415
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
404
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
416
405
  """
417
- Enables loading / saving of models within a step.
406
+ Creates a human-readable report, a Metaflow Card, after this step completes.
407
+
408
+ Note that you may add multiple `@card` decorators in a step with different parameters.
418
409
  """
419
410
  ...
420
411
 
421
412
  @typing.overload
422
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
413
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
423
414
  ...
424
415
 
425
416
  @typing.overload
426
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
417
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
427
418
  ...
428
419
 
429
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
420
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
430
421
  """
431
- Enables loading / saving of models within a step.
422
+ Creates a human-readable report, a Metaflow Card, after this step completes.
423
+
424
+ Note that you may add multiple `@card` decorators in a step with different parameters.
432
425
  """
433
426
  ...
434
427
 
435
- @typing.overload
436
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
428
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
437
429
  """
438
- Specifies environment variables to be set prior to the execution of a step.
430
+ Specifies that this step should execute on Kubernetes.
439
431
  """
440
432
  ...
441
433
 
442
434
  @typing.overload
443
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
444
- ...
445
-
446
- @typing.overload
447
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
448
- ...
449
-
450
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
435
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
451
436
  """
452
- Specifies environment variables to be set prior to the execution of a step.
437
+ Specifies the PyPI packages for the step.
438
+
439
+ Information in this decorator will augment any
440
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
441
+ you can use `@pypi_base` to set packages required by all
442
+ steps and use `@pypi` to specify step-specific overrides.
453
443
  """
454
444
  ...
455
445
 
456
446
  @typing.overload
457
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
458
- """
459
- Specifies secrets to be retrieved and injected as environment variables prior to
460
- the execution of a step.
461
- """
447
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
462
448
  ...
463
449
 
464
450
  @typing.overload
465
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
451
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
466
452
  ...
467
453
 
468
- @typing.overload
469
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
470
- ...
471
-
472
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
454
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
473
455
  """
474
- Specifies secrets to be retrieved and injected as environment variables prior to
475
- the execution of a step.
456
+ Specifies the PyPI packages for the step.
457
+
458
+ Information in this decorator will augment any
459
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
460
+ you can use `@pypi_base` to set packages required by all
461
+ steps and use `@pypi` to specify step-specific overrides.
476
462
  """
477
463
  ...
478
464
 
479
- @typing.overload
480
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
465
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
481
466
  """
482
- Specifies the resources needed when executing this step.
483
-
484
- Use `@resources` to specify the resource requirements
485
- independently of the specific compute layer (`@batch`, `@kubernetes`).
486
-
487
- You can choose the compute layer on the command line by executing e.g.
488
- ```
489
- python myflow.py run --with batch
490
- ```
491
- or
492
- ```
493
- python myflow.py run --with kubernetes
494
- ```
495
- which executes the flow on the desired system using the
496
- requirements specified in `@resources`.
467
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
497
468
  """
498
469
  ...
499
470
 
500
471
  @typing.overload
501
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
472
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
473
+ """
474
+ Decorator prototype for all step decorators. This function gets specialized
475
+ and imported for all decorators types by _import_plugin_decorators().
476
+ """
502
477
  ...
503
478
 
504
479
  @typing.overload
505
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
480
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
506
481
  ...
507
482
 
508
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
483
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
509
484
  """
510
- Specifies the resources needed when executing this step.
511
-
512
- Use `@resources` to specify the resource requirements
513
- independently of the specific compute layer (`@batch`, `@kubernetes`).
514
-
515
- You can choose the compute layer on the command line by executing e.g.
516
- ```
517
- python myflow.py run --with batch
518
- ```
519
- or
520
- ```
521
- python myflow.py run --with kubernetes
522
- ```
523
- which executes the flow on the desired system using the
524
- requirements specified in `@resources`.
485
+ Decorator prototype for all step decorators. This function gets specialized
486
+ and imported for all decorators types by _import_plugin_decorators().
525
487
  """
526
488
  ...
527
489
 
528
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
490
+ @typing.overload
491
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
529
492
  """
530
- Specifies that this step should execute on Kubernetes.
493
+ Specifies environment variables to be set prior to the execution of a step.
531
494
  """
532
495
  ...
533
496
 
534
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
535
- """
536
- Specifies what flows belong to the same project.
537
-
538
- A project-specific namespace is created for all flows that
539
- use the same `@project(name)`.
540
- """
497
+ @typing.overload
498
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
541
499
  ...
542
500
 
543
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
501
+ @typing.overload
502
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
503
+ ...
504
+
505
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
544
506
  """
545
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
546
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
547
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
548
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
549
- starts only after all sensors finish.
507
+ Specifies environment variables to be set prior to the execution of a step.
550
508
  """
551
509
  ...
552
510
 
553
511
  @typing.overload
554
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
512
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
555
513
  """
556
- Specifies the flow(s) that this flow depends on.
557
-
558
- ```
559
- @trigger_on_finish(flow='FooFlow')
560
- ```
561
- or
562
- ```
563
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
564
- ```
565
- This decorator respects the @project decorator and triggers the flow
566
- when upstream runs within the same namespace complete successfully
567
-
568
- Additionally, you can specify project aware upstream flow dependencies
569
- by specifying the fully qualified project_flow_name.
570
- ```
571
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
572
- ```
573
- or
574
- ```
575
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
576
- ```
577
-
578
- You can also specify just the project or project branch (other values will be
579
- inferred from the current project or project branch):
580
- ```
581
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
582
- ```
583
-
584
- Note that `branch` is typically one of:
585
- - `prod`
586
- - `user.bob`
587
- - `test.my_experiment`
588
- - `prod.staging`
514
+ Specifies secrets to be retrieved and injected as environment variables prior to
515
+ the execution of a step.
589
516
  """
590
517
  ...
591
518
 
592
519
  @typing.overload
593
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
520
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
594
521
  ...
595
522
 
596
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
597
- """
598
- Specifies the flow(s) that this flow depends on.
599
-
600
- ```
601
- @trigger_on_finish(flow='FooFlow')
602
- ```
603
- or
604
- ```
605
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
606
- ```
607
- This decorator respects the @project decorator and triggers the flow
608
- when upstream runs within the same namespace complete successfully
609
-
610
- Additionally, you can specify project aware upstream flow dependencies
611
- by specifying the fully qualified project_flow_name.
612
- ```
613
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
614
- ```
615
- or
616
- ```
617
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
618
- ```
619
-
620
- You can also specify just the project or project branch (other values will be
621
- inferred from the current project or project branch):
622
- ```
623
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
624
- ```
625
-
626
- Note that `branch` is typically one of:
627
- - `prod`
628
- - `user.bob`
629
- - `test.my_experiment`
630
- - `prod.staging`
631
- """
523
+ @typing.overload
524
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
632
525
  ...
633
526
 
634
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
527
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
635
528
  """
636
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
637
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
529
+ Specifies secrets to be retrieved and injected as environment variables prior to
530
+ the execution of a step.
638
531
  """
639
532
  ...
640
533
 
641
- @typing.overload
642
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
534
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
643
535
  """
644
- Specifies the times when the flow should be run when running on a
645
- production scheduler.
536
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
537
+
538
+ User code call
539
+ -----------
540
+ @nim(
541
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
542
+ backend='managed'
543
+ )
544
+
545
+ Valid backend options
546
+ ---------------------
547
+ - 'managed': Outerbounds selects a compute provider based on the model.
548
+ - 🚧 'dataplane': Run in your account.
549
+
550
+ Valid model options
551
+ ----------------
552
+ - 'meta/llama3-8b-instruct': 8B parameter model
553
+ - 'meta/llama3-70b-instruct': 70B parameter model
554
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
646
555
  """
647
556
  ...
648
557
 
649
- @typing.overload
650
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
558
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
559
+ """
560
+ Specifies what flows belong to the same project.
561
+
562
+ A project-specific namespace is created for all flows that
563
+ use the same `@project(name)`.
564
+ """
651
565
  ...
652
566
 
653
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
567
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
654
568
  """
655
- Specifies the times when the flow should be run when running on a
656
- production scheduler.
569
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
570
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
571
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
572
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
573
+ starts only after all sensors finish.
657
574
  """
658
575
  ...
659
576
 
@@ -753,30 +670,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
753
670
  """
754
671
  ...
755
672
 
756
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
757
- """
758
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
759
-
760
- User code call
761
- -----------
762
- @nim(
763
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
764
- backend='managed'
765
- )
766
-
767
- Valid backend options
768
- ---------------------
769
- - 'managed': Outerbounds selects a compute provider based on the model.
770
- - 🚧 'dataplane': Run in your account.
771
-
772
- Valid model options
773
- ----------------
774
- - 'meta/llama3-8b-instruct': 8B parameter model
775
- - 'meta/llama3-70b-instruct': 70B parameter model
776
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
777
- """
778
- ...
779
-
780
673
  @typing.overload
781
674
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
782
675
  """
@@ -800,5 +693,112 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
800
693
  """
801
694
  ...
802
695
 
696
+ @typing.overload
697
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
698
+ """
699
+ Specifies the flow(s) that this flow depends on.
700
+
701
+ ```
702
+ @trigger_on_finish(flow='FooFlow')
703
+ ```
704
+ or
705
+ ```
706
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
707
+ ```
708
+ This decorator respects the @project decorator and triggers the flow
709
+ when upstream runs within the same namespace complete successfully
710
+
711
+ Additionally, you can specify project aware upstream flow dependencies
712
+ by specifying the fully qualified project_flow_name.
713
+ ```
714
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
715
+ ```
716
+ or
717
+ ```
718
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
719
+ ```
720
+
721
+ You can also specify just the project or project branch (other values will be
722
+ inferred from the current project or project branch):
723
+ ```
724
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
725
+ ```
726
+
727
+ Note that `branch` is typically one of:
728
+ - `prod`
729
+ - `user.bob`
730
+ - `test.my_experiment`
731
+ - `prod.staging`
732
+ """
733
+ ...
734
+
735
+ @typing.overload
736
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
737
+ ...
738
+
739
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
740
+ """
741
+ Specifies the flow(s) that this flow depends on.
742
+
743
+ ```
744
+ @trigger_on_finish(flow='FooFlow')
745
+ ```
746
+ or
747
+ ```
748
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
749
+ ```
750
+ This decorator respects the @project decorator and triggers the flow
751
+ when upstream runs within the same namespace complete successfully
752
+
753
+ Additionally, you can specify project aware upstream flow dependencies
754
+ by specifying the fully qualified project_flow_name.
755
+ ```
756
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
757
+ ```
758
+ or
759
+ ```
760
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
761
+ ```
762
+
763
+ You can also specify just the project or project branch (other values will be
764
+ inferred from the current project or project branch):
765
+ ```
766
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
767
+ ```
768
+
769
+ Note that `branch` is typically one of:
770
+ - `prod`
771
+ - `user.bob`
772
+ - `test.my_experiment`
773
+ - `prod.staging`
774
+ """
775
+ ...
776
+
777
+ @typing.overload
778
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
779
+ """
780
+ Specifies the times when the flow should be run when running on a
781
+ production scheduler.
782
+ """
783
+ ...
784
+
785
+ @typing.overload
786
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
787
+ ...
788
+
789
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
790
+ """
791
+ Specifies the times when the flow should be run when running on a
792
+ production scheduler.
793
+ """
794
+ ...
795
+
796
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
797
+ """
798
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
799
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
800
+ """
801
+ ...
802
+
803
803
  pkg_name: str
804
804