ob-metaflow-stubs 6.0.3.135__py2.py3-none-any.whl → 6.0.3.137__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (192) hide show
  1. metaflow-stubs/__init__.pyi +278 -278
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +6 -6
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +5 -5
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +98 -98
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +3 -3
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +5 -5
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +5 -5
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +4 -4
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +4 -4
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +4 -4
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +5 -5
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  61. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  62. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  63. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  70. metaflow-stubs/multicore_utils.pyi +2 -2
  71. metaflow-stubs/parameters.pyi +5 -5
  72. metaflow-stubs/plugins/__init__.pyi +11 -11
  73. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  74. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  75. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  76. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  77. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  78. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  79. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  80. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  82. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  83. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  84. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  85. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  86. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -4
  87. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  88. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  89. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  90. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  92. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  93. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  94. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  95. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  96. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  97. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  98. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  99. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  100. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  101. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  102. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +4 -4
  103. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  105. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  106. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  107. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  108. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  109. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  110. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  111. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  112. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  113. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  114. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  116. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  117. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  118. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  119. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  120. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  121. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  123. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  124. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  125. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  126. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  127. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  128. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  129. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  130. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  131. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  132. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  133. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  134. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  135. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  136. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  138. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  139. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  140. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  141. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  142. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  143. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  144. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  146. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  147. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  148. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  149. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  150. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  151. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  152. metaflow-stubs/plugins/perimeters.pyi +2 -2
  153. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  154. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  155. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  156. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  157. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  158. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  159. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  160. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  161. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  162. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  163. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  164. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  165. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  166. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  167. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  168. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  169. metaflow-stubs/profilers/__init__.pyi +2 -2
  170. metaflow-stubs/pylint_wrapper.pyi +2 -2
  171. metaflow-stubs/runner/__init__.pyi +2 -2
  172. metaflow-stubs/runner/deployer.pyi +6 -6
  173. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  174. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  175. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  176. metaflow-stubs/runner/nbrun.pyi +2 -2
  177. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  178. metaflow-stubs/runner/utils.pyi +4 -4
  179. metaflow-stubs/system/__init__.pyi +2 -2
  180. metaflow-stubs/system/system_logger.pyi +3 -3
  181. metaflow-stubs/system/system_monitor.pyi +2 -2
  182. metaflow-stubs/tagging_util.pyi +2 -2
  183. metaflow-stubs/tuple_util.pyi +2 -2
  184. metaflow-stubs/user_configs/__init__.pyi +2 -2
  185. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  186. metaflow-stubs/user_configs/config_options.pyi +4 -4
  187. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  188. {ob_metaflow_stubs-6.0.3.135.dist-info → ob_metaflow_stubs-6.0.3.137.dist-info}/METADATA +1 -1
  189. ob_metaflow_stubs-6.0.3.137.dist-info/RECORD +192 -0
  190. ob_metaflow_stubs-6.0.3.135.dist-info/RECORD +0 -192
  191. {ob_metaflow_stubs-6.0.3.135.dist-info → ob_metaflow_stubs-6.0.3.137.dist-info}/WHEEL +0 -0
  192. {ob_metaflow_stubs-6.0.3.135.dist-info → ob_metaflow_stubs-6.0.3.137.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.13.8.1+obcheckpoint(0.1.6);ob(v1) #
4
- # Generated on 2025-01-30T21:59:34.749405 #
3
+ # MF version: 2.13.9.1+obcheckpoint(0.1.7);ob(v1) #
4
+ # Generated on 2025-02-03T17:09:26.636126 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -35,9 +35,9 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
+ from . import cards as cards
38
39
  from . import events as events
39
40
  from . import tuple_util as tuple_util
40
- from . import cards as cards
41
41
  from . import runner as runner
42
42
  from . import plugins as plugins
43
43
  from .mf_extensions.outerbounds.toplevel.global_aliases_for_metaflow_package import S3 as S3
@@ -147,91 +147,54 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
147
147
  ...
148
148
 
149
149
  @typing.overload
150
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
150
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
151
151
  """
152
- Specifies the Conda environment for the step.
153
-
154
- Information in this decorator will augment any
155
- attributes set in the `@conda_base` flow-level decorator. Hence,
156
- you can use `@conda_base` to set packages required by all
157
- steps and use `@conda` to specify step-specific overrides.
152
+ Specifies environment variables to be set prior to the execution of a step.
158
153
  """
159
154
  ...
160
155
 
161
156
  @typing.overload
162
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
157
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
163
158
  ...
164
159
 
165
160
  @typing.overload
166
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
167
- ...
168
-
169
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
170
- """
171
- Specifies the Conda environment for the step.
172
-
173
- Information in this decorator will augment any
174
- attributes set in the `@conda_base` flow-level decorator. Hence,
175
- you can use `@conda_base` to set packages required by all
176
- steps and use `@conda` to specify step-specific overrides.
177
- """
161
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
178
162
  ...
179
163
 
180
- def nvidia(*, gpu: int, gpu_type: str) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
164
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
181
165
  """
182
- Specifies that this step should execute on DGX cloud.
166
+ Specifies environment variables to be set prior to the execution of a step.
183
167
  """
184
168
  ...
185
169
 
186
170
  @typing.overload
187
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
171
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
188
172
  """
189
- Specifies that the step will success under all circumstances.
173
+ Specifies the Conda environment for the step.
190
174
 
191
- The decorator will create an optional artifact, specified by `var`, which
192
- contains the exception raised. You can use it to detect the presence
193
- of errors, indicating that all happy-path artifacts produced by the step
194
- are missing.
175
+ Information in this decorator will augment any
176
+ attributes set in the `@conda_base` flow-level decorator. Hence,
177
+ you can use `@conda_base` to set packages required by all
178
+ steps and use `@conda` to specify step-specific overrides.
195
179
  """
196
180
  ...
197
181
 
198
182
  @typing.overload
199
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
183
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
200
184
  ...
201
185
 
202
186
  @typing.overload
203
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
187
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
204
188
  ...
205
189
 
206
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
190
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
207
191
  """
208
- Specifies that the step will success under all circumstances.
192
+ Specifies the Conda environment for the step.
209
193
 
210
- The decorator will create an optional artifact, specified by `var`, which
211
- contains the exception raised. You can use it to detect the presence
212
- of errors, indicating that all happy-path artifacts produced by the step
213
- are missing.
214
- """
215
- ...
216
-
217
- @typing.overload
218
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
219
- """
220
- Enables checkpointing for a step.
221
- """
222
- ...
223
-
224
- @typing.overload
225
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
226
- ...
227
-
228
- @typing.overload
229
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
230
- ...
231
-
232
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
233
- """
234
- Enables checkpointing for a step.
194
+ Information in this decorator will augment any
195
+ attributes set in the `@conda_base` flow-level decorator. Hence,
196
+ you can use `@conda_base` to set packages required by all
197
+ steps and use `@conda` to specify step-specific overrides.
235
198
  """
236
199
  ...
237
200
 
@@ -284,6 +247,37 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
284
247
  """
285
248
  ...
286
249
 
250
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
251
+ """
252
+ Specifies that this step should execute on Kubernetes.
253
+ """
254
+ ...
255
+
256
+ @typing.overload
257
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
258
+ """
259
+ Creates a human-readable report, a Metaflow Card, after this step completes.
260
+
261
+ Note that you may add multiple `@card` decorators in a step with different parameters.
262
+ """
263
+ ...
264
+
265
+ @typing.overload
266
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
267
+ ...
268
+
269
+ @typing.overload
270
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
271
+ ...
272
+
273
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
274
+ """
275
+ Creates a human-readable report, a Metaflow Card, after this step completes.
276
+
277
+ Note that you may add multiple `@card` decorators in a step with different parameters.
278
+ """
279
+ ...
280
+
287
281
  @typing.overload
288
282
  def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
289
283
  """
@@ -324,62 +318,54 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
324
318
  ...
325
319
 
326
320
  @typing.overload
327
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
321
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
328
322
  """
329
- Specifies the number of times the task corresponding
330
- to a step needs to be retried.
331
-
332
- This decorator is useful for handling transient errors, such as networking issues.
333
- If your task contains operations that can't be retried safely, e.g. database updates,
334
- it is advisable to annotate it with `@retry(times=0)`.
335
-
336
- This can be used in conjunction with the `@catch` decorator. The `@catch`
337
- decorator will execute a no-op task after all retries have been exhausted,
338
- ensuring that the flow execution can continue.
323
+ Enables checkpointing for a step.
339
324
  """
340
325
  ...
341
326
 
342
327
  @typing.overload
343
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
328
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
344
329
  ...
345
330
 
346
331
  @typing.overload
347
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
332
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
348
333
  ...
349
334
 
350
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
335
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
351
336
  """
352
- Specifies the number of times the task corresponding
353
- to a step needs to be retried.
354
-
355
- This decorator is useful for handling transient errors, such as networking issues.
356
- If your task contains operations that can't be retried safely, e.g. database updates,
357
- it is advisable to annotate it with `@retry(times=0)`.
358
-
359
- This can be used in conjunction with the `@catch` decorator. The `@catch`
360
- decorator will execute a no-op task after all retries have been exhausted,
361
- ensuring that the flow execution can continue.
337
+ Enables checkpointing for a step.
362
338
  """
363
339
  ...
364
340
 
365
341
  @typing.overload
366
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
342
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
367
343
  """
368
- Enables loading / saving of models within a step.
344
+ Specifies the PyPI packages for the step.
345
+
346
+ Information in this decorator will augment any
347
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
348
+ you can use `@pypi_base` to set packages required by all
349
+ steps and use `@pypi` to specify step-specific overrides.
369
350
  """
370
351
  ...
371
352
 
372
353
  @typing.overload
373
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
354
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
374
355
  ...
375
356
 
376
357
  @typing.overload
377
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
358
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
378
359
  ...
379
360
 
380
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
361
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
381
362
  """
382
- Enables loading / saving of models within a step.
363
+ Specifies the PyPI packages for the step.
364
+
365
+ Information in this decorator will augment any
366
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
367
+ you can use `@pypi_base` to set packages required by all
368
+ steps and use `@pypi` to specify step-specific overrides.
383
369
  """
384
370
  ...
385
371
 
@@ -401,64 +387,64 @@ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepF
401
387
  ...
402
388
 
403
389
  @typing.overload
404
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
390
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
405
391
  """
406
- Creates a human-readable report, a Metaflow Card, after this step completes.
392
+ Specifies the number of times the task corresponding
393
+ to a step needs to be retried.
407
394
 
408
- Note that you may add multiple `@card` decorators in a step with different parameters.
395
+ This decorator is useful for handling transient errors, such as networking issues.
396
+ If your task contains operations that can't be retried safely, e.g. database updates,
397
+ it is advisable to annotate it with `@retry(times=0)`.
398
+
399
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
400
+ decorator will execute a no-op task after all retries have been exhausted,
401
+ ensuring that the flow execution can continue.
409
402
  """
410
403
  ...
411
404
 
412
405
  @typing.overload
413
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
406
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
414
407
  ...
415
408
 
416
409
  @typing.overload
417
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
410
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
418
411
  ...
419
412
 
420
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
413
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
421
414
  """
422
- Creates a human-readable report, a Metaflow Card, after this step completes.
415
+ Specifies the number of times the task corresponding
416
+ to a step needs to be retried.
423
417
 
424
- Note that you may add multiple `@card` decorators in a step with different parameters.
425
- """
426
- ...
427
-
428
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
429
- """
430
- Specifies that this step should execute on Kubernetes.
418
+ This decorator is useful for handling transient errors, such as networking issues.
419
+ If your task contains operations that can't be retried safely, e.g. database updates,
420
+ it is advisable to annotate it with `@retry(times=0)`.
421
+
422
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
423
+ decorator will execute a no-op task after all retries have been exhausted,
424
+ ensuring that the flow execution can continue.
431
425
  """
432
426
  ...
433
427
 
434
428
  @typing.overload
435
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
429
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
436
430
  """
437
- Specifies the PyPI packages for the step.
438
-
439
- Information in this decorator will augment any
440
- attributes set in the `@pyi_base` flow-level decorator. Hence,
441
- you can use `@pypi_base` to set packages required by all
442
- steps and use `@pypi` to specify step-specific overrides.
431
+ Specifies secrets to be retrieved and injected as environment variables prior to
432
+ the execution of a step.
443
433
  """
444
434
  ...
445
435
 
446
436
  @typing.overload
447
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
437
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
448
438
  ...
449
439
 
450
440
  @typing.overload
451
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
441
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
452
442
  ...
453
443
 
454
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
444
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
455
445
  """
456
- Specifies the PyPI packages for the step.
457
-
458
- Information in this decorator will augment any
459
- attributes set in the `@pyi_base` flow-level decorator. Hence,
460
- you can use `@pypi_base` to set packages required by all
461
- steps and use `@pypi` to specify step-specific overrides.
446
+ Specifies secrets to be retrieved and injected as environment variables prior to
447
+ the execution of a step.
462
448
  """
463
449
  ...
464
450
 
@@ -469,65 +455,200 @@ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.
469
455
  ...
470
456
 
471
457
  @typing.overload
472
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
458
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
473
459
  """
474
- Decorator prototype for all step decorators. This function gets specialized
475
- and imported for all decorators types by _import_plugin_decorators().
460
+ Enables loading / saving of models within a step.
476
461
  """
477
462
  ...
478
463
 
479
464
  @typing.overload
480
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
465
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
481
466
  ...
482
467
 
483
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
468
+ @typing.overload
469
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
470
+ ...
471
+
472
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
484
473
  """
485
- Decorator prototype for all step decorators. This function gets specialized
486
- and imported for all decorators types by _import_plugin_decorators().
474
+ Enables loading / saving of models within a step.
487
475
  """
488
476
  ...
489
477
 
490
478
  @typing.overload
491
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
479
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
492
480
  """
493
- Specifies environment variables to be set prior to the execution of a step.
481
+ Specifies that the step will success under all circumstances.
482
+
483
+ The decorator will create an optional artifact, specified by `var`, which
484
+ contains the exception raised. You can use it to detect the presence
485
+ of errors, indicating that all happy-path artifacts produced by the step
486
+ are missing.
494
487
  """
495
488
  ...
496
489
 
497
490
  @typing.overload
498
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
491
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
499
492
  ...
500
493
 
501
494
  @typing.overload
502
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
495
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
503
496
  ...
504
497
 
505
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
506
- """
507
- Specifies environment variables to be set prior to the execution of a step.
498
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
508
499
  """
509
- ...
510
-
500
+ Specifies that the step will success under all circumstances.
501
+
502
+ The decorator will create an optional artifact, specified by `var`, which
503
+ contains the exception raised. You can use it to detect the presence
504
+ of errors, indicating that all happy-path artifacts produced by the step
505
+ are missing.
506
+ """
507
+ ...
508
+
509
+ def nvidia(*, gpu: int, gpu_type: str, queue_timeout: int) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
510
+ """
511
+ Specifies that this step should execute on DGX cloud.
512
+ """
513
+ ...
514
+
511
515
  @typing.overload
512
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
516
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
513
517
  """
514
- Specifies secrets to be retrieved and injected as environment variables prior to
515
- the execution of a step.
518
+ Decorator prototype for all step decorators. This function gets specialized
519
+ and imported for all decorators types by _import_plugin_decorators().
516
520
  """
517
521
  ...
518
522
 
519
523
  @typing.overload
520
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
524
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
525
+ ...
526
+
527
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
528
+ """
529
+ Decorator prototype for all step decorators. This function gets specialized
530
+ and imported for all decorators types by _import_plugin_decorators().
531
+ """
521
532
  ...
522
533
 
523
534
  @typing.overload
524
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
535
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
536
+ """
537
+ Specifies the PyPI packages for all steps of the flow.
538
+
539
+ Use `@pypi_base` to set common packages required by all
540
+ steps and use `@pypi` to specify step-specific overrides.
541
+ """
525
542
  ...
526
543
 
527
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
544
+ @typing.overload
545
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
546
+ ...
547
+
548
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
528
549
  """
529
- Specifies secrets to be retrieved and injected as environment variables prior to
530
- the execution of a step.
550
+ Specifies the PyPI packages for all steps of the flow.
551
+
552
+ Use `@pypi_base` to set common packages required by all
553
+ steps and use `@pypi` to specify step-specific overrides.
554
+ """
555
+ ...
556
+
557
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
558
+ """
559
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
560
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
561
+ """
562
+ ...
563
+
564
+ @typing.overload
565
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
566
+ """
567
+ Specifies the flow(s) that this flow depends on.
568
+
569
+ ```
570
+ @trigger_on_finish(flow='FooFlow')
571
+ ```
572
+ or
573
+ ```
574
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
575
+ ```
576
+ This decorator respects the @project decorator and triggers the flow
577
+ when upstream runs within the same namespace complete successfully
578
+
579
+ Additionally, you can specify project aware upstream flow dependencies
580
+ by specifying the fully qualified project_flow_name.
581
+ ```
582
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
583
+ ```
584
+ or
585
+ ```
586
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
587
+ ```
588
+
589
+ You can also specify just the project or project branch (other values will be
590
+ inferred from the current project or project branch):
591
+ ```
592
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
593
+ ```
594
+
595
+ Note that `branch` is typically one of:
596
+ - `prod`
597
+ - `user.bob`
598
+ - `test.my_experiment`
599
+ - `prod.staging`
600
+ """
601
+ ...
602
+
603
+ @typing.overload
604
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
605
+ ...
606
+
607
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
608
+ """
609
+ Specifies the flow(s) that this flow depends on.
610
+
611
+ ```
612
+ @trigger_on_finish(flow='FooFlow')
613
+ ```
614
+ or
615
+ ```
616
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
617
+ ```
618
+ This decorator respects the @project decorator and triggers the flow
619
+ when upstream runs within the same namespace complete successfully
620
+
621
+ Additionally, you can specify project aware upstream flow dependencies
622
+ by specifying the fully qualified project_flow_name.
623
+ ```
624
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
625
+ ```
626
+ or
627
+ ```
628
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
629
+ ```
630
+
631
+ You can also specify just the project or project branch (other values will be
632
+ inferred from the current project or project branch):
633
+ ```
634
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
635
+ ```
636
+
637
+ Note that `branch` is typically one of:
638
+ - `prod`
639
+ - `user.bob`
640
+ - `test.my_experiment`
641
+ - `prod.staging`
642
+ """
643
+ ...
644
+
645
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
646
+ """
647
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
648
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
649
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
650
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
651
+ starts only after all sensors finish.
531
652
  """
532
653
  ...
533
654
 
@@ -555,45 +676,31 @@ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[Fl
555
676
  """
556
677
  ...
557
678
 
558
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
679
+ @typing.overload
680
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
559
681
  """
560
- Specifies what flows belong to the same project.
561
-
562
- A project-specific namespace is created for all flows that
563
- use the same `@project(name)`.
682
+ Specifies the times when the flow should be run when running on a
683
+ production scheduler.
564
684
  """
565
685
  ...
566
686
 
567
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
568
- """
569
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
570
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
571
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
572
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
573
- starts only after all sensors finish.
574
- """
687
+ @typing.overload
688
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
575
689
  ...
576
690
 
577
- @typing.overload
578
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
691
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
579
692
  """
580
- Specifies the PyPI packages for all steps of the flow.
581
-
582
- Use `@pypi_base` to set common packages required by all
583
- steps and use `@pypi` to specify step-specific overrides.
693
+ Specifies the times when the flow should be run when running on a
694
+ production scheduler.
584
695
  """
585
696
  ...
586
697
 
587
- @typing.overload
588
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
589
- ...
590
-
591
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
698
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
592
699
  """
593
- Specifies the PyPI packages for all steps of the flow.
700
+ Specifies what flows belong to the same project.
594
701
 
595
- Use `@pypi_base` to set common packages required by all
596
- steps and use `@pypi` to specify step-specific overrides.
702
+ A project-specific namespace is created for all flows that
703
+ use the same `@project(name)`.
597
704
  """
598
705
  ...
599
706
 
@@ -693,112 +800,5 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
693
800
  """
694
801
  ...
695
802
 
696
- @typing.overload
697
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
698
- """
699
- Specifies the flow(s) that this flow depends on.
700
-
701
- ```
702
- @trigger_on_finish(flow='FooFlow')
703
- ```
704
- or
705
- ```
706
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
707
- ```
708
- This decorator respects the @project decorator and triggers the flow
709
- when upstream runs within the same namespace complete successfully
710
-
711
- Additionally, you can specify project aware upstream flow dependencies
712
- by specifying the fully qualified project_flow_name.
713
- ```
714
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
715
- ```
716
- or
717
- ```
718
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
719
- ```
720
-
721
- You can also specify just the project or project branch (other values will be
722
- inferred from the current project or project branch):
723
- ```
724
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
725
- ```
726
-
727
- Note that `branch` is typically one of:
728
- - `prod`
729
- - `user.bob`
730
- - `test.my_experiment`
731
- - `prod.staging`
732
- """
733
- ...
734
-
735
- @typing.overload
736
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
737
- ...
738
-
739
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
740
- """
741
- Specifies the flow(s) that this flow depends on.
742
-
743
- ```
744
- @trigger_on_finish(flow='FooFlow')
745
- ```
746
- or
747
- ```
748
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
749
- ```
750
- This decorator respects the @project decorator and triggers the flow
751
- when upstream runs within the same namespace complete successfully
752
-
753
- Additionally, you can specify project aware upstream flow dependencies
754
- by specifying the fully qualified project_flow_name.
755
- ```
756
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
757
- ```
758
- or
759
- ```
760
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
761
- ```
762
-
763
- You can also specify just the project or project branch (other values will be
764
- inferred from the current project or project branch):
765
- ```
766
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
767
- ```
768
-
769
- Note that `branch` is typically one of:
770
- - `prod`
771
- - `user.bob`
772
- - `test.my_experiment`
773
- - `prod.staging`
774
- """
775
- ...
776
-
777
- @typing.overload
778
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
779
- """
780
- Specifies the times when the flow should be run when running on a
781
- production scheduler.
782
- """
783
- ...
784
-
785
- @typing.overload
786
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
787
- ...
788
-
789
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
790
- """
791
- Specifies the times when the flow should be run when running on a
792
- production scheduler.
793
- """
794
- ...
795
-
796
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
797
- """
798
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
799
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
800
- """
801
- ...
802
-
803
803
  pkg_name: str
804
804