ob-metaflow-stubs 6.0.3.121__py2.py3-none-any.whl → 6.0.3.122__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (174) hide show
  1. metaflow-stubs/__init__.pyi +233 -233
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +2 -2
  5. metaflow-stubs/client/core.pyi +6 -6
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +4 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  16. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  17. metaflow-stubs/metadata_provider/util.pyi +2 -2
  18. metaflow-stubs/metaflow_config.pyi +4 -2
  19. metaflow-stubs/metaflow_current.pyi +34 -34
  20. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  21. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +3 -3
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +3 -3
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +2 -2
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +4 -4
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  42. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  43. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  44. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  45. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  46. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  47. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  48. metaflow-stubs/mflog/__init__.pyi +2 -2
  49. metaflow-stubs/mflog/mflog.pyi +2 -2
  50. metaflow-stubs/multicore_utils.pyi +11 -11
  51. metaflow-stubs/parameters.pyi +4 -4
  52. metaflow-stubs/plugins/__init__.pyi +12 -12
  53. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  54. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  55. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  56. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  58. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  59. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  60. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  61. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  62. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  63. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -3
  64. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +2 -2
  65. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  66. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +4 -4
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -3
  68. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  69. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  70. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  71. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  73. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  74. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  75. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  76. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  77. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  79. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +5 -4
  85. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  92. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  93. metaflow-stubs/plugins/cards/card_cli.pyi +2 -2
  94. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  95. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  96. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  97. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  100. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  101. metaflow-stubs/plugins/cards/card_modules/components.pyi +5 -5
  102. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  103. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  104. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  105. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  106. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  107. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  108. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  109. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  110. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  111. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  112. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  113. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  114. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  115. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  116. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  117. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/events_decorator.pyi +12 -2
  119. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  120. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  121. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  122. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  123. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  124. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  125. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  126. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  127. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  128. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +8 -2
  129. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +5 -5
  130. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  131. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  132. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +8 -2
  133. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +3 -2
  134. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  135. metaflow-stubs/plugins/package_cli.pyi +2 -2
  136. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  137. metaflow-stubs/plugins/perimeters.pyi +2 -2
  138. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  139. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  140. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  141. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  142. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  143. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  144. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  145. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  146. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  147. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  148. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  149. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  150. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  151. metaflow-stubs/plugins/tag_cli.pyi +2 -2
  152. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  153. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  154. metaflow-stubs/procpoll.pyi +2 -2
  155. metaflow-stubs/profilers/__init__.pyi +2 -2
  156. metaflow-stubs/pylint_wrapper.pyi +2 -2
  157. metaflow-stubs/runner/__init__.pyi +3 -3
  158. metaflow-stubs/runner/deployer.pyi +30 -30
  159. metaflow-stubs/runner/deployer_impl.pyi +3 -2
  160. metaflow-stubs/runner/metaflow_runner.pyi +7 -3
  161. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  162. metaflow-stubs/runner/nbrun.pyi +2 -2
  163. metaflow-stubs/runner/subprocess_manager.pyi +7 -3
  164. metaflow-stubs/runner/utils.pyi +109 -10
  165. metaflow-stubs/system/__init__.pyi +2 -2
  166. metaflow-stubs/system/system_logger.pyi +3 -3
  167. metaflow-stubs/system/system_monitor.pyi +2 -2
  168. metaflow-stubs/tagging_util.pyi +2 -2
  169. metaflow-stubs/tuple_util.pyi +2 -2
  170. {ob_metaflow_stubs-6.0.3.121.dist-info → ob_metaflow_stubs-6.0.3.122.dist-info}/METADATA +1 -1
  171. ob_metaflow_stubs-6.0.3.122.dist-info/RECORD +174 -0
  172. ob_metaflow_stubs-6.0.3.121.dist-info/RECORD +0 -174
  173. {ob_metaflow_stubs-6.0.3.121.dist-info → ob_metaflow_stubs-6.0.3.122.dist-info}/WHEEL +0 -0
  174. {ob_metaflow_stubs-6.0.3.121.dist-info → ob_metaflow_stubs-6.0.3.122.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.33.1+obcheckpoint(0.1.4);ob(v1) #
4
- # Generated on 2024-12-02T17:17:46.476932 #
3
+ # MF version: 2.12.35.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-12-06T18:19:55.683902 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -140,6 +140,72 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
140
140
  """
141
141
  ...
142
142
 
143
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
144
+ """
145
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
146
+ """
147
+ ...
148
+
149
+ @typing.overload
150
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
151
+ """
152
+ Specifies the number of times the task corresponding
153
+ to a step needs to be retried.
154
+
155
+ This decorator is useful for handling transient errors, such as networking issues.
156
+ If your task contains operations that can't be retried safely, e.g. database updates,
157
+ it is advisable to annotate it with `@retry(times=0)`.
158
+
159
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
160
+ decorator will execute a no-op task after all retries have been exhausted,
161
+ ensuring that the flow execution can continue.
162
+ """
163
+ ...
164
+
165
+ @typing.overload
166
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
167
+ ...
168
+
169
+ @typing.overload
170
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
171
+ ...
172
+
173
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
174
+ """
175
+ Specifies the number of times the task corresponding
176
+ to a step needs to be retried.
177
+
178
+ This decorator is useful for handling transient errors, such as networking issues.
179
+ If your task contains operations that can't be retried safely, e.g. database updates,
180
+ it is advisable to annotate it with `@retry(times=0)`.
181
+
182
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
183
+ decorator will execute a no-op task after all retries have been exhausted,
184
+ ensuring that the flow execution can continue.
185
+ """
186
+ ...
187
+
188
+ @typing.overload
189
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
190
+ """
191
+ Specifies environment variables to be set prior to the execution of a step.
192
+ """
193
+ ...
194
+
195
+ @typing.overload
196
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
197
+ ...
198
+
199
+ @typing.overload
200
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
201
+ ...
202
+
203
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
204
+ """
205
+ Specifies environment variables to be set prior to the execution of a step.
206
+ """
207
+ ...
208
+
143
209
  @typing.overload
144
210
  def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
145
211
  """
@@ -160,33 +226,33 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
160
226
  ...
161
227
 
162
228
  @typing.overload
163
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
229
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
164
230
  """
165
- Specifies the Conda environment for the step.
231
+ Specifies the PyPI packages for the step.
166
232
 
167
233
  Information in this decorator will augment any
168
- attributes set in the `@conda_base` flow-level decorator. Hence,
169
- you can use `@conda_base` to set packages required by all
170
- steps and use `@conda` to specify step-specific overrides.
234
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
235
+ you can use `@pypi_base` to set packages required by all
236
+ steps and use `@pypi` to specify step-specific overrides.
171
237
  """
172
238
  ...
173
239
 
174
240
  @typing.overload
175
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
241
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
176
242
  ...
177
243
 
178
244
  @typing.overload
179
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
245
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
180
246
  ...
181
247
 
182
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
248
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
183
249
  """
184
- Specifies the Conda environment for the step.
250
+ Specifies the PyPI packages for the step.
185
251
 
186
252
  Information in this decorator will augment any
187
- attributes set in the `@conda_base` flow-level decorator. Hence,
188
- you can use `@conda_base` to set packages required by all
189
- steps and use `@conda` to specify step-specific overrides.
253
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
254
+ you can use `@pypi_base` to set packages required by all
255
+ steps and use `@pypi` to specify step-specific overrides.
190
256
  """
191
257
  ...
192
258
 
@@ -239,9 +305,24 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
239
305
  """
240
306
  ...
241
307
 
242
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
308
+ @typing.overload
309
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
243
310
  """
244
- Decorator that helps cache, version and store models/datasets from huggingface hub.
311
+ Enables checkpointing for a step.
312
+ """
313
+ ...
314
+
315
+ @typing.overload
316
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
317
+ ...
318
+
319
+ @typing.overload
320
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
321
+ ...
322
+
323
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
324
+ """
325
+ Enables checkpointing for a step.
245
326
  """
246
327
  ...
247
328
 
@@ -277,201 +358,126 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
277
358
  ...
278
359
 
279
360
  @typing.overload
280
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
361
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
281
362
  """
282
- Specifies a timeout for your step.
283
-
284
- This decorator is useful if this step may hang indefinitely.
285
-
286
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
287
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
288
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
363
+ Creates a human-readable report, a Metaflow Card, after this step completes.
289
364
 
290
- Note that all the values specified in parameters are added together so if you specify
291
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
365
+ Note that you may add multiple `@card` decorators in a step with different parameters.
292
366
  """
293
367
  ...
294
368
 
295
369
  @typing.overload
296
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
370
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
297
371
  ...
298
372
 
299
373
  @typing.overload
300
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
374
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
301
375
  ...
302
376
 
303
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
377
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
304
378
  """
305
- Specifies a timeout for your step.
306
-
307
- This decorator is useful if this step may hang indefinitely.
308
-
309
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
310
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
311
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
379
+ Creates a human-readable report, a Metaflow Card, after this step completes.
312
380
 
313
- Note that all the values specified in parameters are added together so if you specify
314
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
315
- """
316
- ...
317
-
318
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
319
- """
320
- Specifies that this step should execute on Kubernetes.
381
+ Note that you may add multiple `@card` decorators in a step with different parameters.
321
382
  """
322
383
  ...
323
384
 
324
385
  @typing.overload
325
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
386
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
326
387
  """
327
- Enables checkpointing for a step.
388
+ Specifies secrets to be retrieved and injected as environment variables prior to
389
+ the execution of a step.
328
390
  """
329
391
  ...
330
392
 
331
393
  @typing.overload
332
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
394
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
333
395
  ...
334
396
 
335
397
  @typing.overload
336
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
398
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
337
399
  ...
338
400
 
339
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
401
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
340
402
  """
341
- Enables checkpointing for a step.
403
+ Specifies secrets to be retrieved and injected as environment variables prior to
404
+ the execution of a step.
342
405
  """
343
406
  ...
344
407
 
345
408
  @typing.overload
346
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
409
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
347
410
  """
348
- Specifies the PyPI packages for the step.
411
+ Specifies the Conda environment for the step.
349
412
 
350
413
  Information in this decorator will augment any
351
- attributes set in the `@pyi_base` flow-level decorator. Hence,
352
- you can use `@pypi_base` to set packages required by all
353
- steps and use `@pypi` to specify step-specific overrides.
414
+ attributes set in the `@conda_base` flow-level decorator. Hence,
415
+ you can use `@conda_base` to set packages required by all
416
+ steps and use `@conda` to specify step-specific overrides.
354
417
  """
355
418
  ...
356
419
 
357
420
  @typing.overload
358
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
421
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
359
422
  ...
360
423
 
361
424
  @typing.overload
362
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
425
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
363
426
  ...
364
427
 
365
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
428
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
366
429
  """
367
- Specifies the PyPI packages for the step.
430
+ Specifies the Conda environment for the step.
368
431
 
369
432
  Information in this decorator will augment any
370
- attributes set in the `@pyi_base` flow-level decorator. Hence,
371
- you can use `@pypi_base` to set packages required by all
372
- steps and use `@pypi` to specify step-specific overrides.
373
- """
374
- ...
375
-
376
- @typing.overload
377
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
378
- """
379
- Specifies environment variables to be set prior to the execution of a step.
380
- """
381
- ...
382
-
383
- @typing.overload
384
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
385
- ...
386
-
387
- @typing.overload
388
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
389
- ...
390
-
391
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
392
- """
393
- Specifies environment variables to be set prior to the execution of a step.
433
+ attributes set in the `@conda_base` flow-level decorator. Hence,
434
+ you can use `@conda_base` to set packages required by all
435
+ steps and use `@conda` to specify step-specific overrides.
394
436
  """
395
437
  ...
396
438
 
397
439
  @typing.overload
398
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
440
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
399
441
  """
400
- Specifies the number of times the task corresponding
401
- to a step needs to be retried.
442
+ Specifies a timeout for your step.
402
443
 
403
- This decorator is useful for handling transient errors, such as networking issues.
404
- If your task contains operations that can't be retried safely, e.g. database updates,
405
- it is advisable to annotate it with `@retry(times=0)`.
444
+ This decorator is useful if this step may hang indefinitely.
406
445
 
407
- This can be used in conjunction with the `@catch` decorator. The `@catch`
408
- decorator will execute a no-op task after all retries have been exhausted,
409
- ensuring that the flow execution can continue.
446
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
447
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
448
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
449
+
450
+ Note that all the values specified in parameters are added together so if you specify
451
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
410
452
  """
411
453
  ...
412
454
 
413
455
  @typing.overload
414
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
456
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
415
457
  ...
416
458
 
417
459
  @typing.overload
418
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
460
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
419
461
  ...
420
462
 
421
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
463
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
422
464
  """
423
- Specifies the number of times the task corresponding
424
- to a step needs to be retried.
425
-
426
- This decorator is useful for handling transient errors, such as networking issues.
427
- If your task contains operations that can't be retried safely, e.g. database updates,
428
- it is advisable to annotate it with `@retry(times=0)`.
465
+ Specifies a timeout for your step.
429
466
 
430
- This can be used in conjunction with the `@catch` decorator. The `@catch`
431
- decorator will execute a no-op task after all retries have been exhausted,
432
- ensuring that the flow execution can continue.
433
- """
434
- ...
435
-
436
- @typing.overload
437
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
438
- """
439
- Creates a human-readable report, a Metaflow Card, after this step completes.
467
+ This decorator is useful if this step may hang indefinitely.
440
468
 
441
- Note that you may add multiple `@card` decorators in a step with different parameters.
442
- """
443
- ...
444
-
445
- @typing.overload
446
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
447
- ...
448
-
449
- @typing.overload
450
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
451
- ...
452
-
453
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
454
- """
455
- Creates a human-readable report, a Metaflow Card, after this step completes.
469
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
470
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
471
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
456
472
 
457
- Note that you may add multiple `@card` decorators in a step with different parameters.
458
- """
459
- ...
460
-
461
- @typing.overload
462
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
463
- """
464
- Internal decorator to support Fast bakery
473
+ Note that all the values specified in parameters are added together so if you specify
474
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
465
475
  """
466
476
  ...
467
477
 
468
- @typing.overload
469
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
470
- ...
471
-
472
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
478
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
473
479
  """
474
- Internal decorator to support Fast bakery
480
+ Specifies that this step should execute on Kubernetes.
475
481
  """
476
482
  ...
477
483
 
@@ -497,91 +503,19 @@ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
497
503
  ...
498
504
 
499
505
  @typing.overload
500
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
501
- """
502
- Specifies secrets to be retrieved and injected as environment variables prior to
503
- the execution of a step.
504
- """
505
- ...
506
-
507
- @typing.overload
508
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
509
- ...
510
-
511
- @typing.overload
512
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
513
- ...
514
-
515
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
516
- """
517
- Specifies secrets to be retrieved and injected as environment variables prior to
518
- the execution of a step.
519
- """
520
- ...
521
-
522
- @typing.overload
523
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
524
- """
525
- Specifies the PyPI packages for all steps of the flow.
526
-
527
- Use `@pypi_base` to set common packages required by all
528
- steps and use `@pypi` to specify step-specific overrides.
529
- """
530
- ...
531
-
532
- @typing.overload
533
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
534
- ...
535
-
536
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
537
- """
538
- Specifies the PyPI packages for all steps of the flow.
539
-
540
- Use `@pypi_base` to set common packages required by all
541
- steps and use `@pypi` to specify step-specific overrides.
542
- """
543
- ...
544
-
545
- @typing.overload
546
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
506
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
547
507
  """
548
- Specifies the times when the flow should be run when running on a
549
- production scheduler.
508
+ Internal decorator to support Fast bakery
550
509
  """
551
510
  ...
552
511
 
553
512
  @typing.overload
554
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
555
- ...
556
-
557
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
558
- """
559
- Specifies the times when the flow should be run when running on a
560
- production scheduler.
561
- """
513
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
562
514
  ...
563
515
 
564
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
516
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
565
517
  """
566
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
567
-
568
- User code call
569
- -----------
570
- @nim(
571
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
572
- backend='managed'
573
- )
574
-
575
- Valid backend options
576
- ---------------------
577
- - 'managed': Outerbounds selects a compute provider based on the model.
578
- - 🚧 'dataplane': Run in your account.
579
-
580
- Valid model options
581
- ----------------
582
- - 'meta/llama3-8b-instruct': 8B parameter model
583
- - 'meta/llama3-70b-instruct': 70B parameter model
584
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
518
+ Internal decorator to support Fast bakery
585
519
  """
586
520
  ...
587
521
 
@@ -666,13 +600,59 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
666
600
  """
667
601
  ...
668
602
 
669
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
603
+ @typing.overload
604
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
670
605
  """
671
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
672
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
673
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
674
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
675
- starts only after all sensors finish.
606
+ Specifies the PyPI packages for all steps of the flow.
607
+
608
+ Use `@pypi_base` to set common packages required by all
609
+ steps and use `@pypi` to specify step-specific overrides.
610
+ """
611
+ ...
612
+
613
+ @typing.overload
614
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
615
+ ...
616
+
617
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
618
+ """
619
+ Specifies the PyPI packages for all steps of the flow.
620
+
621
+ Use `@pypi_base` to set common packages required by all
622
+ steps and use `@pypi` to specify step-specific overrides.
623
+ """
624
+ ...
625
+
626
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
627
+ """
628
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
629
+
630
+ User code call
631
+ -----------
632
+ @nim(
633
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
634
+ backend='managed'
635
+ )
636
+
637
+ Valid backend options
638
+ ---------------------
639
+ - 'managed': Outerbounds selects a compute provider based on the model.
640
+ - 🚧 'dataplane': Run in your account.
641
+
642
+ Valid model options
643
+ ----------------
644
+ - 'meta/llama3-8b-instruct': 8B parameter model
645
+ - 'meta/llama3-70b-instruct': 70B parameter model
646
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
647
+ """
648
+ ...
649
+
650
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
651
+ """
652
+ Specifies what flows belong to the same project.
653
+
654
+ A project-specific namespace is created for all flows that
655
+ use the same `@project(name)`.
676
656
  """
677
657
  ...
678
658
 
@@ -749,6 +729,35 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
749
729
  """
750
730
  ...
751
731
 
732
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
733
+ """
734
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
735
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
736
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
737
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
738
+ starts only after all sensors finish.
739
+ """
740
+ ...
741
+
742
+ @typing.overload
743
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
744
+ """
745
+ Specifies the times when the flow should be run when running on a
746
+ production scheduler.
747
+ """
748
+ ...
749
+
750
+ @typing.overload
751
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
752
+ ...
753
+
754
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
755
+ """
756
+ Specifies the times when the flow should be run when running on a
757
+ production scheduler.
758
+ """
759
+ ...
760
+
752
761
  def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
753
762
  """
754
763
  The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
@@ -779,14 +788,5 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
779
788
  """
780
789
  ...
781
790
 
782
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
783
- """
784
- Specifies what flows belong to the same project.
785
-
786
- A project-specific namespace is created for all flows that
787
- use the same `@project(name)`.
788
- """
789
- ...
790
-
791
791
  pkg_name: str
792
792