ob-metaflow-stubs 6.0.3.119__py2.py3-none-any.whl → 6.0.3.120__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (174) hide show
  1. metaflow-stubs/__init__.pyi +252 -252
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +2 -2
  5. metaflow-stubs/client/core.pyi +4 -4
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +4 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +3 -3
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  16. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  17. metaflow-stubs/metadata_provider/util.pyi +2 -2
  18. metaflow-stubs/metaflow_config.pyi +2 -2
  19. metaflow-stubs/metaflow_current.pyi +87 -87
  20. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  21. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  42. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  43. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  44. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  45. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  46. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  47. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  48. metaflow-stubs/mflog/__init__.pyi +2 -2
  49. metaflow-stubs/mflog/mflog.pyi +2 -2
  50. metaflow-stubs/multicore_utils.pyi +2 -2
  51. metaflow-stubs/parameters.pyi +3 -3
  52. metaflow-stubs/plugins/__init__.pyi +13 -13
  53. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  54. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  55. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  56. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  58. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  59. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  60. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  61. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  62. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  63. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  64. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +2 -2
  65. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  66. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  68. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  69. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  70. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  71. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  73. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  74. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  75. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  76. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  77. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  79. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  85. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  92. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  93. metaflow-stubs/plugins/cards/card_cli.pyi +2 -2
  94. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  95. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  96. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  97. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  100. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  101. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  102. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  103. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  104. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  105. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  106. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  107. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  108. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  109. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  110. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  111. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  112. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  113. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  114. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  115. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  116. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  117. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/events_decorator.pyi +12 -2
  119. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  120. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  121. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  122. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  123. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  124. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  125. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  126. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  127. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  128. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  129. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  130. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  131. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  132. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  133. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  134. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  135. metaflow-stubs/plugins/package_cli.pyi +2 -2
  136. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  137. metaflow-stubs/plugins/perimeters.pyi +2 -2
  138. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  139. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  140. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  141. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  142. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  143. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  144. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  145. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  146. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  147. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  148. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  149. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  150. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  151. metaflow-stubs/plugins/tag_cli.pyi +2 -2
  152. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  153. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  154. metaflow-stubs/procpoll.pyi +2 -2
  155. metaflow-stubs/profilers/__init__.pyi +2 -2
  156. metaflow-stubs/pylint_wrapper.pyi +2 -2
  157. metaflow-stubs/runner/__init__.pyi +2 -2
  158. metaflow-stubs/runner/deployer.pyi +6 -6
  159. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  160. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  161. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  162. metaflow-stubs/runner/nbrun.pyi +2 -2
  163. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  164. metaflow-stubs/runner/utils.pyi +4 -4
  165. metaflow-stubs/system/__init__.pyi +2 -2
  166. metaflow-stubs/system/system_logger.pyi +3 -3
  167. metaflow-stubs/system/system_monitor.pyi +2 -2
  168. metaflow-stubs/tagging_util.pyi +2 -2
  169. metaflow-stubs/tuple_util.pyi +2 -2
  170. {ob_metaflow_stubs-6.0.3.119.dist-info → ob_metaflow_stubs-6.0.3.120.dist-info}/METADATA +1 -1
  171. ob_metaflow_stubs-6.0.3.120.dist-info/RECORD +174 -0
  172. {ob_metaflow_stubs-6.0.3.119.dist-info → ob_metaflow_stubs-6.0.3.120.dist-info}/WHEEL +1 -1
  173. ob_metaflow_stubs-6.0.3.119.dist-info/RECORD +0 -174
  174. {ob_metaflow_stubs-6.0.3.119.dist-info → ob_metaflow_stubs-6.0.3.120.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
4
- # Generated on 2024-11-21T22:12:20.897909 #
3
+ # MF version: 2.12.32.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-26T19:30:33.284631 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -29,8 +29,8 @@ from .flowspec import FlowSpec as FlowSpec
29
29
  from .parameters import Parameter as Parameter
30
30
  from .parameters import JSONTypeClass as JSONTypeClass
31
31
  from .parameters import JSONType as JSONType
32
- from . import tuple_util as tuple_util
33
32
  from . import cards as cards
33
+ from . import tuple_util as tuple_util
34
34
  from . import events as events
35
35
  from . import runner as runner
36
36
  from . import plugins as plugins
@@ -140,48 +140,42 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
140
140
  """
141
141
  ...
142
142
 
143
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
144
- """
145
- Specifies that this step should execute on Kubernetes.
146
- """
147
- ...
148
-
149
143
  @typing.overload
150
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
144
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
151
145
  """
152
- Specifies a timeout for your step.
153
-
154
- This decorator is useful if this step may hang indefinitely.
146
+ Specifies the number of times the task corresponding
147
+ to a step needs to be retried.
155
148
 
156
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
157
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
158
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
149
+ This decorator is useful for handling transient errors, such as networking issues.
150
+ If your task contains operations that can't be retried safely, e.g. database updates,
151
+ it is advisable to annotate it with `@retry(times=0)`.
159
152
 
160
- Note that all the values specified in parameters are added together so if you specify
161
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
153
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
154
+ decorator will execute a no-op task after all retries have been exhausted,
155
+ ensuring that the flow execution can continue.
162
156
  """
163
157
  ...
164
158
 
165
159
  @typing.overload
166
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
160
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
167
161
  ...
168
162
 
169
163
  @typing.overload
170
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
164
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
171
165
  ...
172
166
 
173
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
167
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
174
168
  """
175
- Specifies a timeout for your step.
176
-
177
- This decorator is useful if this step may hang indefinitely.
169
+ Specifies the number of times the task corresponding
170
+ to a step needs to be retried.
178
171
 
179
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
180
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
181
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
172
+ This decorator is useful for handling transient errors, such as networking issues.
173
+ If your task contains operations that can't be retried safely, e.g. database updates,
174
+ it is advisable to annotate it with `@retry(times=0)`.
182
175
 
183
- Note that all the values specified in parameters are added together so if you specify
184
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
176
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
177
+ decorator will execute a no-op task after all retries have been exhausted,
178
+ ensuring that the flow execution can continue.
185
179
  """
186
180
  ...
187
181
 
@@ -235,19 +229,33 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
235
229
  ...
236
230
 
237
231
  @typing.overload
238
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
232
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
239
233
  """
240
- Internal decorator to support Fast bakery
234
+ Specifies the Conda environment for the step.
235
+
236
+ Information in this decorator will augment any
237
+ attributes set in the `@conda_base` flow-level decorator. Hence,
238
+ you can use `@conda_base` to set packages required by all
239
+ steps and use `@conda` to specify step-specific overrides.
241
240
  """
242
241
  ...
243
242
 
244
243
  @typing.overload
245
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
244
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
246
245
  ...
247
246
 
248
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
247
+ @typing.overload
248
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
249
+ ...
250
+
251
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
249
252
  """
250
- Internal decorator to support Fast bakery
253
+ Specifies the Conda environment for the step.
254
+
255
+ Information in this decorator will augment any
256
+ attributes set in the `@conda_base` flow-level decorator. Hence,
257
+ you can use `@conda_base` to set packages required by all
258
+ steps and use `@conda` to specify step-specific overrides.
251
259
  """
252
260
  ...
253
261
 
@@ -283,87 +291,54 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
283
291
  ...
284
292
 
285
293
  @typing.overload
286
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
287
- """
288
- Enables loading / saving of models within a step.
289
- """
290
- ...
291
-
292
- @typing.overload
293
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
294
- ...
295
-
296
- @typing.overload
297
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
298
- ...
299
-
300
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
301
- """
302
- Enables loading / saving of models within a step.
303
- """
304
- ...
305
-
306
- @typing.overload
307
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
294
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
308
295
  """
309
- Creates a human-readable report, a Metaflow Card, after this step completes.
296
+ Specifies the PyPI packages for the step.
310
297
 
311
- Note that you may add multiple `@card` decorators in a step with different parameters.
298
+ Information in this decorator will augment any
299
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
300
+ you can use `@pypi_base` to set packages required by all
301
+ steps and use `@pypi` to specify step-specific overrides.
312
302
  """
313
303
  ...
314
304
 
315
305
  @typing.overload
316
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
306
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
317
307
  ...
318
308
 
319
309
  @typing.overload
320
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
310
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
321
311
  ...
322
312
 
323
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
313
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
324
314
  """
325
- Creates a human-readable report, a Metaflow Card, after this step completes.
315
+ Specifies the PyPI packages for the step.
326
316
 
327
- Note that you may add multiple `@card` decorators in a step with different parameters.
317
+ Information in this decorator will augment any
318
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
319
+ you can use `@pypi_base` to set packages required by all
320
+ steps and use `@pypi` to specify step-specific overrides.
328
321
  """
329
322
  ...
330
323
 
331
324
  @typing.overload
332
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
325
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
333
326
  """
334
- Specifies the number of times the task corresponding
335
- to a step needs to be retried.
336
-
337
- This decorator is useful for handling transient errors, such as networking issues.
338
- If your task contains operations that can't be retried safely, e.g. database updates,
339
- it is advisable to annotate it with `@retry(times=0)`.
340
-
341
- This can be used in conjunction with the `@catch` decorator. The `@catch`
342
- decorator will execute a no-op task after all retries have been exhausted,
343
- ensuring that the flow execution can continue.
327
+ Specifies environment variables to be set prior to the execution of a step.
344
328
  """
345
329
  ...
346
330
 
347
331
  @typing.overload
348
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
332
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
349
333
  ...
350
334
 
351
335
  @typing.overload
352
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
336
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
353
337
  ...
354
338
 
355
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
339
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
356
340
  """
357
- Specifies the number of times the task corresponding
358
- to a step needs to be retried.
359
-
360
- This decorator is useful for handling transient errors, such as networking issues.
361
- If your task contains operations that can't be retried safely, e.g. database updates,
362
- it is advisable to annotate it with `@retry(times=0)`.
363
-
364
- This can be used in conjunction with the `@catch` decorator. The `@catch`
365
- decorator will execute a no-op task after all retries have been exhausted,
366
- ensuring that the flow execution can continue.
341
+ Specifies environment variables to be set prior to the execution of a step.
367
342
  """
368
343
  ...
369
344
 
@@ -388,65 +363,57 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
388
363
  """
389
364
  ...
390
365
 
391
- @typing.overload
392
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
366
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
393
367
  """
394
- Specifies the PyPI packages for the step.
395
-
396
- Information in this decorator will augment any
397
- attributes set in the `@pyi_base` flow-level decorator. Hence,
398
- you can use `@pypi_base` to set packages required by all
399
- steps and use `@pypi` to specify step-specific overrides.
368
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
400
369
  """
401
370
  ...
402
371
 
403
372
  @typing.overload
404
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
373
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
374
+ """
375
+ Internal decorator to support Fast bakery
376
+ """
405
377
  ...
406
378
 
407
379
  @typing.overload
408
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
380
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
409
381
  ...
410
382
 
411
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
383
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
412
384
  """
413
- Specifies the PyPI packages for the step.
414
-
415
- Information in this decorator will augment any
416
- attributes set in the `@pyi_base` flow-level decorator. Hence,
417
- you can use `@pypi_base` to set packages required by all
418
- steps and use `@pypi` to specify step-specific overrides.
385
+ Internal decorator to support Fast bakery
386
+ """
387
+ ...
388
+
389
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
390
+ """
391
+ Specifies that this step should execute on Kubernetes.
419
392
  """
420
393
  ...
421
394
 
422
395
  @typing.overload
423
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
396
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
424
397
  """
425
- Specifies the Conda environment for the step.
398
+ Creates a human-readable report, a Metaflow Card, after this step completes.
426
399
 
427
- Information in this decorator will augment any
428
- attributes set in the `@conda_base` flow-level decorator. Hence,
429
- you can use `@conda_base` to set packages required by all
430
- steps and use `@conda` to specify step-specific overrides.
400
+ Note that you may add multiple `@card` decorators in a step with different parameters.
431
401
  """
432
402
  ...
433
403
 
434
404
  @typing.overload
435
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
405
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
436
406
  ...
437
407
 
438
408
  @typing.overload
439
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
409
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
440
410
  ...
441
411
 
442
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
412
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
443
413
  """
444
- Specifies the Conda environment for the step.
414
+ Creates a human-readable report, a Metaflow Card, after this step completes.
445
415
 
446
- Information in this decorator will augment any
447
- attributes set in the `@conda_base` flow-level decorator. Hence,
448
- you can use `@conda_base` to set packages required by all
449
- steps and use `@conda` to specify step-specific overrides.
416
+ Note that you may add multiple `@card` decorators in a step with different parameters.
450
417
  """
451
418
  ...
452
419
 
@@ -473,12 +440,6 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
473
440
  """
474
441
  ...
475
442
 
476
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
477
- """
478
- Decorator that helps cache, version and store models/datasets from huggingface hub.
479
- """
480
- ...
481
-
482
443
  @typing.overload
483
444
  def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
484
445
  """
@@ -499,87 +460,169 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
499
460
  ...
500
461
 
501
462
  @typing.overload
502
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
463
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
503
464
  """
504
- Specifies environment variables to be set prior to the execution of a step.
465
+ Enables loading / saving of models within a step.
505
466
  """
506
467
  ...
507
468
 
508
469
  @typing.overload
509
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
470
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
510
471
  ...
511
472
 
512
473
  @typing.overload
513
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
474
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
514
475
  ...
515
476
 
516
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
477
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
517
478
  """
518
- Specifies environment variables to be set prior to the execution of a step.
479
+ Enables loading / saving of models within a step.
519
480
  """
520
481
  ...
521
482
 
522
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
483
+ @typing.overload
484
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
523
485
  """
524
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
525
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
486
+ Specifies a timeout for your step.
487
+
488
+ This decorator is useful if this step may hang indefinitely.
489
+
490
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
491
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
492
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
493
+
494
+ Note that all the values specified in parameters are added together so if you specify
495
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
526
496
  """
527
497
  ...
528
498
 
529
499
  @typing.overload
530
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
500
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
501
+ ...
502
+
503
+ @typing.overload
504
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
505
+ ...
506
+
507
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
531
508
  """
532
- Specifies the PyPI packages for all steps of the flow.
509
+ Specifies a timeout for your step.
533
510
 
534
- Use `@pypi_base` to set common packages required by all
535
- steps and use `@pypi` to specify step-specific overrides.
511
+ This decorator is useful if this step may hang indefinitely.
512
+
513
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
514
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
515
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
516
+
517
+ Note that all the values specified in parameters are added together so if you specify
518
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
536
519
  """
537
520
  ...
538
521
 
539
522
  @typing.overload
540
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
523
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
524
+ """
525
+ Specifies the times when the flow should be run when running on a
526
+ production scheduler.
527
+ """
541
528
  ...
542
529
 
543
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
530
+ @typing.overload
531
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
532
+ ...
533
+
534
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
544
535
  """
545
- Specifies the PyPI packages for all steps of the flow.
536
+ Specifies the times when the flow should be run when running on a
537
+ production scheduler.
538
+ """
539
+ ...
540
+
541
+ @typing.overload
542
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
543
+ """
544
+ Specifies the flow(s) that this flow depends on.
546
545
 
547
- Use `@pypi_base` to set common packages required by all
548
- steps and use `@pypi` to specify step-specific overrides.
546
+ ```
547
+ @trigger_on_finish(flow='FooFlow')
548
+ ```
549
+ or
550
+ ```
551
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
552
+ ```
553
+ This decorator respects the @project decorator and triggers the flow
554
+ when upstream runs within the same namespace complete successfully
555
+
556
+ Additionally, you can specify project aware upstream flow dependencies
557
+ by specifying the fully qualified project_flow_name.
558
+ ```
559
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
560
+ ```
561
+ or
562
+ ```
563
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
564
+ ```
565
+
566
+ You can also specify just the project or project branch (other values will be
567
+ inferred from the current project or project branch):
568
+ ```
569
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
570
+ ```
571
+
572
+ Note that `branch` is typically one of:
573
+ - `prod`
574
+ - `user.bob`
575
+ - `test.my_experiment`
576
+ - `prod.staging`
549
577
  """
550
578
  ...
551
579
 
552
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
580
+ @typing.overload
581
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
582
+ ...
583
+
584
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
553
585
  """
554
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
586
+ Specifies the flow(s) that this flow depends on.
555
587
 
556
- User code call
557
- -----------
558
- @nim(
559
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
560
- backend='managed'
561
- )
588
+ ```
589
+ @trigger_on_finish(flow='FooFlow')
590
+ ```
591
+ or
592
+ ```
593
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
594
+ ```
595
+ This decorator respects the @project decorator and triggers the flow
596
+ when upstream runs within the same namespace complete successfully
562
597
 
563
- Valid backend options
564
- ---------------------
565
- - 'managed': Outerbounds selects a compute provider based on the model.
566
- - 🚧 'dataplane': Run in your account.
598
+ Additionally, you can specify project aware upstream flow dependencies
599
+ by specifying the fully qualified project_flow_name.
600
+ ```
601
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
602
+ ```
603
+ or
604
+ ```
605
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
606
+ ```
567
607
 
568
- Valid model options
569
- ----------------
570
- - 'meta/llama3-8b-instruct': 8B parameter model
571
- - 'meta/llama3-70b-instruct': 70B parameter model
572
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
608
+ You can also specify just the project or project branch (other values will be
609
+ inferred from the current project or project branch):
610
+ ```
611
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
612
+ ```
613
+
614
+ Note that `branch` is typically one of:
615
+ - `prod`
616
+ - `user.bob`
617
+ - `test.my_experiment`
618
+ - `prod.staging`
573
619
  """
574
620
  ...
575
621
 
576
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
622
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
577
623
  """
578
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
579
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
580
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
581
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
582
- starts only after all sensors finish.
624
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
625
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
583
626
  """
584
627
  ...
585
628
 
@@ -657,125 +700,82 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
657
700
  ...
658
701
 
659
702
  @typing.overload
660
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
703
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
661
704
  """
662
- Specifies the Conda environment for all steps of the flow.
705
+ Specifies the PyPI packages for all steps of the flow.
663
706
 
664
- Use `@conda_base` to set common libraries required by all
665
- steps and use `@conda` to specify step-specific additions.
707
+ Use `@pypi_base` to set common packages required by all
708
+ steps and use `@pypi` to specify step-specific overrides.
666
709
  """
667
710
  ...
668
711
 
669
712
  @typing.overload
670
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
713
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
671
714
  ...
672
715
 
673
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
716
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
674
717
  """
675
- Specifies the Conda environment for all steps of the flow.
718
+ Specifies the PyPI packages for all steps of the flow.
676
719
 
677
- Use `@conda_base` to set common libraries required by all
678
- steps and use `@conda` to specify step-specific additions.
720
+ Use `@pypi_base` to set common packages required by all
721
+ steps and use `@pypi` to specify step-specific overrides.
679
722
  """
680
723
  ...
681
724
 
682
725
  @typing.overload
683
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
726
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
684
727
  """
685
- Specifies the flow(s) that this flow depends on.
686
-
687
- ```
688
- @trigger_on_finish(flow='FooFlow')
689
- ```
690
- or
691
- ```
692
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
693
- ```
694
- This decorator respects the @project decorator and triggers the flow
695
- when upstream runs within the same namespace complete successfully
696
-
697
- Additionally, you can specify project aware upstream flow dependencies
698
- by specifying the fully qualified project_flow_name.
699
- ```
700
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
701
- ```
702
- or
703
- ```
704
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
705
- ```
706
-
707
- You can also specify just the project or project branch (other values will be
708
- inferred from the current project or project branch):
709
- ```
710
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
711
- ```
728
+ Specifies the Conda environment for all steps of the flow.
712
729
 
713
- Note that `branch` is typically one of:
714
- - `prod`
715
- - `user.bob`
716
- - `test.my_experiment`
717
- - `prod.staging`
730
+ Use `@conda_base` to set common libraries required by all
731
+ steps and use `@conda` to specify step-specific additions.
718
732
  """
719
733
  ...
720
734
 
721
735
  @typing.overload
722
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
736
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
723
737
  ...
724
738
 
725
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
739
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
726
740
  """
727
- Specifies the flow(s) that this flow depends on.
728
-
729
- ```
730
- @trigger_on_finish(flow='FooFlow')
731
- ```
732
- or
733
- ```
734
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
735
- ```
736
- This decorator respects the @project decorator and triggers the flow
737
- when upstream runs within the same namespace complete successfully
738
-
739
- Additionally, you can specify project aware upstream flow dependencies
740
- by specifying the fully qualified project_flow_name.
741
- ```
742
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
743
- ```
744
- or
745
- ```
746
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
747
- ```
748
-
749
- You can also specify just the project or project branch (other values will be
750
- inferred from the current project or project branch):
751
- ```
752
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
753
- ```
741
+ Specifies the Conda environment for all steps of the flow.
754
742
 
755
- Note that `branch` is typically one of:
756
- - `prod`
757
- - `user.bob`
758
- - `test.my_experiment`
759
- - `prod.staging`
743
+ Use `@conda_base` to set common libraries required by all
744
+ steps and use `@conda` to specify step-specific additions.
760
745
  """
761
746
  ...
762
747
 
763
- @typing.overload
764
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
748
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
765
749
  """
766
- Specifies the times when the flow should be run when running on a
767
- production scheduler.
750
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
751
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
752
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
753
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
754
+ starts only after all sensors finish.
768
755
  """
769
756
  ...
770
757
 
771
- @typing.overload
772
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
773
- ...
774
-
775
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
758
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
776
759
  """
777
- Specifies the times when the flow should be run when running on a
778
- production scheduler.
760
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
761
+
762
+ User code call
763
+ -----------
764
+ @nim(
765
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
766
+ backend='managed'
767
+ )
768
+
769
+ Valid backend options
770
+ ---------------------
771
+ - 'managed': Outerbounds selects a compute provider based on the model.
772
+ - 🚧 'dataplane': Run in your account.
773
+
774
+ Valid model options
775
+ ----------------
776
+ - 'meta/llama3-8b-instruct': 8B parameter model
777
+ - 'meta/llama3-70b-instruct': 70B parameter model
778
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
779
779
  """
780
780
  ...
781
781