ob-metaflow-stubs 6.0.3.122__py2.py3-none-any.whl → 6.0.3.123__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (174) hide show
  1. metaflow-stubs/__init__.pyi +241 -241
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +2 -2
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +4 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +3 -3
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  16. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  17. metaflow-stubs/metadata_provider/util.pyi +2 -2
  18. metaflow-stubs/metaflow_config.pyi +2 -2
  19. metaflow-stubs/metaflow_current.pyi +87 -87
  20. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  21. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +4 -4
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +4 -4
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +3 -3
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +3 -3
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +2 -2
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +2 -2
  42. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  43. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  44. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  45. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  46. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  47. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  48. metaflow-stubs/mflog/__init__.pyi +2 -2
  49. metaflow-stubs/mflog/mflog.pyi +2 -2
  50. metaflow-stubs/multicore_utils.pyi +2 -2
  51. metaflow-stubs/parameters.pyi +3 -3
  52. metaflow-stubs/plugins/__init__.pyi +11 -11
  53. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  54. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  55. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  56. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  58. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  59. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  60. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  61. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  62. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  63. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  64. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +2 -2
  65. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  66. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  68. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  69. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  70. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  71. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  73. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  74. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  75. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  76. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  77. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  79. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  85. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  92. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  93. metaflow-stubs/plugins/cards/card_cli.pyi +2 -2
  94. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  95. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  96. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  97. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  100. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  101. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  102. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  103. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  104. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  105. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  106. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  107. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  108. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  109. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  110. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  111. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  112. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  113. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  114. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  115. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  116. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  117. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  119. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  120. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  121. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  122. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  123. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  124. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  125. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  126. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  127. metaflow-stubs/plugins/kubernetes/__init__.pyi +3 -3
  128. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  129. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  130. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  131. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  132. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  133. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  134. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  135. metaflow-stubs/plugins/package_cli.pyi +2 -2
  136. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  137. metaflow-stubs/plugins/perimeters.pyi +2 -2
  138. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  139. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  140. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  141. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  142. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  143. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  144. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  145. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  146. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  147. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  148. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  149. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  150. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  151. metaflow-stubs/plugins/tag_cli.pyi +2 -2
  152. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  153. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  154. metaflow-stubs/procpoll.pyi +2 -2
  155. metaflow-stubs/profilers/__init__.pyi +2 -2
  156. metaflow-stubs/pylint_wrapper.pyi +2 -2
  157. metaflow-stubs/runner/__init__.pyi +2 -2
  158. metaflow-stubs/runner/deployer.pyi +5 -5
  159. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  160. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  161. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  162. metaflow-stubs/runner/nbrun.pyi +2 -2
  163. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  164. metaflow-stubs/runner/utils.pyi +3 -3
  165. metaflow-stubs/system/__init__.pyi +2 -2
  166. metaflow-stubs/system/system_logger.pyi +3 -3
  167. metaflow-stubs/system/system_monitor.pyi +2 -2
  168. metaflow-stubs/tagging_util.pyi +2 -2
  169. metaflow-stubs/tuple_util.pyi +2 -2
  170. {ob_metaflow_stubs-6.0.3.122.dist-info → ob_metaflow_stubs-6.0.3.123.dist-info}/METADATA +1 -1
  171. ob_metaflow_stubs-6.0.3.123.dist-info/RECORD +174 -0
  172. ob_metaflow_stubs-6.0.3.122.dist-info/RECORD +0 -174
  173. {ob_metaflow_stubs-6.0.3.122.dist-info → ob_metaflow_stubs-6.0.3.123.dist-info}/WHEEL +0 -0
  174. {ob_metaflow_stubs-6.0.3.122.dist-info → ob_metaflow_stubs-6.0.3.123.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.35.1+obcheckpoint(0.1.4);ob(v1) #
4
- # Generated on 2024-12-06T18:19:55.683902 #
3
+ # MF version: 2.12.36.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-12-09T17:36:48.672361 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -140,88 +140,82 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
140
140
  """
141
141
  ...
142
142
 
143
- def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
143
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
144
144
  """
145
- Decorator that helps cache, version and store models/datasets from huggingface hub.
145
+ Specifies that this step should execute on Kubernetes.
146
146
  """
147
147
  ...
148
148
 
149
149
  @typing.overload
150
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
150
+ def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
151
151
  """
152
- Specifies the number of times the task corresponding
153
- to a step needs to be retried.
154
-
155
- This decorator is useful for handling transient errors, such as networking issues.
156
- If your task contains operations that can't be retried safely, e.g. database updates,
157
- it is advisable to annotate it with `@retry(times=0)`.
158
-
159
- This can be used in conjunction with the `@catch` decorator. The `@catch`
160
- decorator will execute a no-op task after all retries have been exhausted,
161
- ensuring that the flow execution can continue.
152
+ Enables checkpointing for a step.
162
153
  """
163
154
  ...
164
155
 
165
156
  @typing.overload
166
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
157
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
167
158
  ...
168
159
 
169
160
  @typing.overload
170
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
161
+ def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
171
162
  ...
172
163
 
173
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
164
+ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
174
165
  """
175
- Specifies the number of times the task corresponding
176
- to a step needs to be retried.
177
-
178
- This decorator is useful for handling transient errors, such as networking issues.
179
- If your task contains operations that can't be retried safely, e.g. database updates,
180
- it is advisable to annotate it with `@retry(times=0)`.
181
-
182
- This can be used in conjunction with the `@catch` decorator. The `@catch`
183
- decorator will execute a no-op task after all retries have been exhausted,
184
- ensuring that the flow execution can continue.
166
+ Enables checkpointing for a step.
185
167
  """
186
168
  ...
187
169
 
188
170
  @typing.overload
189
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
171
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
190
172
  """
191
- Specifies environment variables to be set prior to the execution of a step.
173
+ Specifies the Conda environment for the step.
174
+
175
+ Information in this decorator will augment any
176
+ attributes set in the `@conda_base` flow-level decorator. Hence,
177
+ you can use `@conda_base` to set packages required by all
178
+ steps and use `@conda` to specify step-specific overrides.
192
179
  """
193
180
  ...
194
181
 
195
182
  @typing.overload
196
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
183
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
197
184
  ...
198
185
 
199
186
  @typing.overload
200
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
187
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
201
188
  ...
202
189
 
203
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
190
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
204
191
  """
205
- Specifies environment variables to be set prior to the execution of a step.
192
+ Specifies the Conda environment for the step.
193
+
194
+ Information in this decorator will augment any
195
+ attributes set in the `@conda_base` flow-level decorator. Hence,
196
+ you can use `@conda_base` to set packages required by all
197
+ steps and use `@conda` to specify step-specific overrides.
206
198
  """
207
199
  ...
208
200
 
209
201
  @typing.overload
210
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
202
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
211
203
  """
212
- Decorator prototype for all step decorators. This function gets specialized
213
- and imported for all decorators types by _import_plugin_decorators().
204
+ Specifies environment variables to be set prior to the execution of a step.
214
205
  """
215
206
  ...
216
207
 
217
208
  @typing.overload
218
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
209
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
219
210
  ...
220
211
 
221
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
212
+ @typing.overload
213
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
214
+ ...
215
+
216
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
222
217
  """
223
- Decorator prototype for all step decorators. This function gets specialized
224
- and imported for all decorators types by _import_plugin_decorators().
218
+ Specifies environment variables to be set prior to the execution of a step.
225
219
  """
226
220
  ...
227
221
 
@@ -256,6 +250,37 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
256
250
  """
257
251
  ...
258
252
 
253
+ def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None, load: typing.Union[typing.List[str], typing.List[typing.Tuple[typing.Dict, str]], typing.List[typing.Tuple[str, str]], typing.List[typing.Dict], None]) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
254
+ """
255
+ Decorator that helps cache, version and store models/datasets from huggingface hub.
256
+ """
257
+ ...
258
+
259
+ @typing.overload
260
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
261
+ """
262
+ Creates a human-readable report, a Metaflow Card, after this step completes.
263
+
264
+ Note that you may add multiple `@card` decorators in a step with different parameters.
265
+ """
266
+ ...
267
+
268
+ @typing.overload
269
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
270
+ ...
271
+
272
+ @typing.overload
273
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
274
+ ...
275
+
276
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
277
+ """
278
+ Creates a human-readable report, a Metaflow Card, after this step completes.
279
+
280
+ Note that you may add multiple `@card` decorators in a step with different parameters.
281
+ """
282
+ ...
283
+
259
284
  @typing.overload
260
285
  def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
261
286
  """
@@ -306,23 +331,21 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
306
331
  ...
307
332
 
308
333
  @typing.overload
309
- def checkpoint(*, load_policy: str = 'fresh', temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
334
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
310
335
  """
311
- Enables checkpointing for a step.
336
+ Decorator prototype for all step decorators. This function gets specialized
337
+ and imported for all decorators types by _import_plugin_decorators().
312
338
  """
313
339
  ...
314
340
 
315
341
  @typing.overload
316
- def checkpoint(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
317
- ...
318
-
319
- @typing.overload
320
- def checkpoint(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
342
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
321
343
  ...
322
344
 
323
- def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load_policy: str = 'fresh', temp_dir_root: str = None):
345
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
324
346
  """
325
- Enables checkpointing for a step.
347
+ Decorator prototype for all step decorators. This function gets specialized
348
+ and imported for all decorators types by _import_plugin_decorators().
326
349
  """
327
350
  ...
328
351
 
@@ -358,27 +381,41 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
358
381
  ...
359
382
 
360
383
  @typing.overload
361
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
384
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
362
385
  """
363
- Creates a human-readable report, a Metaflow Card, after this step completes.
386
+ Specifies a timeout for your step.
364
387
 
365
- Note that you may add multiple `@card` decorators in a step with different parameters.
388
+ This decorator is useful if this step may hang indefinitely.
389
+
390
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
391
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
392
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
393
+
394
+ Note that all the values specified in parameters are added together so if you specify
395
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
366
396
  """
367
397
  ...
368
398
 
369
399
  @typing.overload
370
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
400
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
371
401
  ...
372
402
 
373
403
  @typing.overload
374
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
404
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
375
405
  ...
376
406
 
377
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
407
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
378
408
  """
379
- Creates a human-readable report, a Metaflow Card, after this step completes.
409
+ Specifies a timeout for your step.
380
410
 
381
- Note that you may add multiple `@card` decorators in a step with different parameters.
411
+ This decorator is useful if this step may hang indefinitely.
412
+
413
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
414
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
415
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
416
+
417
+ Note that all the values specified in parameters are added together so if you specify
418
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
382
419
  """
383
420
  ...
384
421
 
@@ -406,99 +443,41 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
406
443
  ...
407
444
 
408
445
  @typing.overload
409
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
410
- """
411
- Specifies the Conda environment for the step.
412
-
413
- Information in this decorator will augment any
414
- attributes set in the `@conda_base` flow-level decorator. Hence,
415
- you can use `@conda_base` to set packages required by all
416
- steps and use `@conda` to specify step-specific overrides.
417
- """
418
- ...
419
-
420
- @typing.overload
421
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
422
- ...
423
-
424
- @typing.overload
425
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
426
- ...
427
-
428
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
429
- """
430
- Specifies the Conda environment for the step.
431
-
432
- Information in this decorator will augment any
433
- attributes set in the `@conda_base` flow-level decorator. Hence,
434
- you can use `@conda_base` to set packages required by all
435
- steps and use `@conda` to specify step-specific overrides.
436
- """
437
- ...
438
-
439
- @typing.overload
440
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
446
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
441
447
  """
442
- Specifies a timeout for your step.
443
-
444
- This decorator is useful if this step may hang indefinitely.
448
+ Specifies the number of times the task corresponding
449
+ to a step needs to be retried.
445
450
 
446
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
447
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
448
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
451
+ This decorator is useful for handling transient errors, such as networking issues.
452
+ If your task contains operations that can't be retried safely, e.g. database updates,
453
+ it is advisable to annotate it with `@retry(times=0)`.
449
454
 
450
- Note that all the values specified in parameters are added together so if you specify
451
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
455
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
456
+ decorator will execute a no-op task after all retries have been exhausted,
457
+ ensuring that the flow execution can continue.
452
458
  """
453
459
  ...
454
460
 
455
461
  @typing.overload
456
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
462
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
457
463
  ...
458
464
 
459
465
  @typing.overload
460
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
466
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
461
467
  ...
462
468
 
463
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
469
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
464
470
  """
465
- Specifies a timeout for your step.
466
-
467
- This decorator is useful if this step may hang indefinitely.
471
+ Specifies the number of times the task corresponding
472
+ to a step needs to be retried.
468
473
 
469
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
470
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
471
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
474
+ This decorator is useful for handling transient errors, such as networking issues.
475
+ If your task contains operations that can't be retried safely, e.g. database updates,
476
+ it is advisable to annotate it with `@retry(times=0)`.
472
477
 
473
- Note that all the values specified in parameters are added together so if you specify
474
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
475
- """
476
- ...
477
-
478
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
479
- """
480
- Specifies that this step should execute on Kubernetes.
481
- """
482
- ...
483
-
484
- @typing.overload
485
- def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
486
- """
487
- Enables loading / saving of models within a step.
488
- """
489
- ...
490
-
491
- @typing.overload
492
- def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
493
- ...
494
-
495
- @typing.overload
496
- def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
497
- ...
498
-
499
- def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
500
- """
501
- Enables loading / saving of models within a step.
478
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
479
+ decorator will execute a no-op task after all retries have been exhausted,
480
+ ensuring that the flow execution can continue.
502
481
  """
503
482
  ...
504
483
 
@@ -520,106 +499,23 @@ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepF
520
499
  ...
521
500
 
522
501
  @typing.overload
523
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
524
- """
525
- Specifies the flow(s) that this flow depends on.
526
-
527
- ```
528
- @trigger_on_finish(flow='FooFlow')
529
- ```
530
- or
531
- ```
532
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
533
- ```
534
- This decorator respects the @project decorator and triggers the flow
535
- when upstream runs within the same namespace complete successfully
536
-
537
- Additionally, you can specify project aware upstream flow dependencies
538
- by specifying the fully qualified project_flow_name.
539
- ```
540
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
541
- ```
542
- or
543
- ```
544
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
545
- ```
546
-
547
- You can also specify just the project or project branch (other values will be
548
- inferred from the current project or project branch):
549
- ```
550
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
551
- ```
552
-
553
- Note that `branch` is typically one of:
554
- - `prod`
555
- - `user.bob`
556
- - `test.my_experiment`
557
- - `prod.staging`
558
- """
559
- ...
560
-
561
- @typing.overload
562
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
563
- ...
564
-
565
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
502
+ def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
566
503
  """
567
- Specifies the flow(s) that this flow depends on.
568
-
569
- ```
570
- @trigger_on_finish(flow='FooFlow')
571
- ```
572
- or
573
- ```
574
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
575
- ```
576
- This decorator respects the @project decorator and triggers the flow
577
- when upstream runs within the same namespace complete successfully
578
-
579
- Additionally, you can specify project aware upstream flow dependencies
580
- by specifying the fully qualified project_flow_name.
581
- ```
582
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
583
- ```
584
- or
585
- ```
586
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
587
- ```
588
-
589
- You can also specify just the project or project branch (other values will be
590
- inferred from the current project or project branch):
591
- ```
592
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
593
- ```
594
-
595
- Note that `branch` is typically one of:
596
- - `prod`
597
- - `user.bob`
598
- - `test.my_experiment`
599
- - `prod.staging`
504
+ Enables loading / saving of models within a step.
600
505
  """
601
506
  ...
602
507
 
603
508
  @typing.overload
604
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
605
- """
606
- Specifies the PyPI packages for all steps of the flow.
607
-
608
- Use `@pypi_base` to set common packages required by all
609
- steps and use `@pypi` to specify step-specific overrides.
610
- """
509
+ def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
611
510
  ...
612
511
 
613
512
  @typing.overload
614
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
513
+ def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
615
514
  ...
616
515
 
617
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
516
+ def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
618
517
  """
619
- Specifies the PyPI packages for all steps of the flow.
620
-
621
- Use `@pypi_base` to set common packages required by all
622
- steps and use `@pypi` to specify step-specific overrides.
518
+ Enables loading / saving of models within a step.
623
519
  """
624
520
  ...
625
521
 
@@ -647,15 +543,6 @@ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[Fl
647
543
  """
648
544
  ...
649
545
 
650
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
651
- """
652
- Specifies what flows belong to the same project.
653
-
654
- A project-specific namespace is created for all flows that
655
- use the same `@project(name)`.
656
- """
657
- ...
658
-
659
546
  @typing.overload
660
547
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
661
548
  """
@@ -740,21 +627,25 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
740
627
  ...
741
628
 
742
629
  @typing.overload
743
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
630
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
744
631
  """
745
- Specifies the times when the flow should be run when running on a
746
- production scheduler.
632
+ Specifies the PyPI packages for all steps of the flow.
633
+
634
+ Use `@pypi_base` to set common packages required by all
635
+ steps and use `@pypi` to specify step-specific overrides.
747
636
  """
748
637
  ...
749
638
 
750
639
  @typing.overload
751
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
640
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
752
641
  ...
753
642
 
754
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
643
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
755
644
  """
756
- Specifies the times when the flow should be run when running on a
757
- production scheduler.
645
+ Specifies the PyPI packages for all steps of the flow.
646
+
647
+ Use `@pypi_base` to set common packages required by all
648
+ steps and use `@pypi` to specify step-specific overrides.
758
649
  """
759
650
  ...
760
651
 
@@ -765,6 +656,96 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
765
656
  """
766
657
  ...
767
658
 
659
+ @typing.overload
660
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
661
+ """
662
+ Specifies the flow(s) that this flow depends on.
663
+
664
+ ```
665
+ @trigger_on_finish(flow='FooFlow')
666
+ ```
667
+ or
668
+ ```
669
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
670
+ ```
671
+ This decorator respects the @project decorator and triggers the flow
672
+ when upstream runs within the same namespace complete successfully
673
+
674
+ Additionally, you can specify project aware upstream flow dependencies
675
+ by specifying the fully qualified project_flow_name.
676
+ ```
677
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
678
+ ```
679
+ or
680
+ ```
681
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
682
+ ```
683
+
684
+ You can also specify just the project or project branch (other values will be
685
+ inferred from the current project or project branch):
686
+ ```
687
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
688
+ ```
689
+
690
+ Note that `branch` is typically one of:
691
+ - `prod`
692
+ - `user.bob`
693
+ - `test.my_experiment`
694
+ - `prod.staging`
695
+ """
696
+ ...
697
+
698
+ @typing.overload
699
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
700
+ ...
701
+
702
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
703
+ """
704
+ Specifies the flow(s) that this flow depends on.
705
+
706
+ ```
707
+ @trigger_on_finish(flow='FooFlow')
708
+ ```
709
+ or
710
+ ```
711
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
712
+ ```
713
+ This decorator respects the @project decorator and triggers the flow
714
+ when upstream runs within the same namespace complete successfully
715
+
716
+ Additionally, you can specify project aware upstream flow dependencies
717
+ by specifying the fully qualified project_flow_name.
718
+ ```
719
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
720
+ ```
721
+ or
722
+ ```
723
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
724
+ ```
725
+
726
+ You can also specify just the project or project branch (other values will be
727
+ inferred from the current project or project branch):
728
+ ```
729
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
730
+ ```
731
+
732
+ Note that `branch` is typically one of:
733
+ - `prod`
734
+ - `user.bob`
735
+ - `test.my_experiment`
736
+ - `prod.staging`
737
+ """
738
+ ...
739
+
740
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
741
+ """
742
+ Specifies what flows belong to the same project.
743
+
744
+ A project-specific namespace is created for all flows that
745
+ use the same `@project(name)`.
746
+ """
747
+ ...
748
+
768
749
  @typing.overload
769
750
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
770
751
  """
@@ -788,5 +769,24 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
788
769
  """
789
770
  ...
790
771
 
772
+ @typing.overload
773
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
774
+ """
775
+ Specifies the times when the flow should be run when running on a
776
+ production scheduler.
777
+ """
778
+ ...
779
+
780
+ @typing.overload
781
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
782
+ ...
783
+
784
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
785
+ """
786
+ Specifies the times when the flow should be run when running on a
787
+ production scheduler.
788
+ """
789
+ ...
790
+
791
791
  pkg_name: str
792
792