metaflow-stubs 2.12.32__py2.py3-none-any.whl → 2.12.34__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. metaflow-stubs/__init__.pyi +246 -246
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +2 -2
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +3 -3
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +2 -2
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  16. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  17. metaflow-stubs/metadata_provider/util.pyi +2 -2
  18. metaflow-stubs/metaflow_config.pyi +2 -2
  19. metaflow-stubs/metaflow_current.pyi +18 -18
  20. metaflow-stubs/mflog/__init__.pyi +2 -2
  21. metaflow-stubs/mflog/mflog.pyi +2 -2
  22. metaflow-stubs/multicore_utils.pyi +11 -11
  23. metaflow-stubs/parameters.pyi +2 -2
  24. metaflow-stubs/plugins/__init__.pyi +12 -12
  25. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/airflow.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  31. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  33. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  34. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  35. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  37. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  38. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  39. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +2 -2
  40. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  41. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  42. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -3
  43. metaflow-stubs/plugins/aws/__init__.pyi +3 -3
  44. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  46. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  47. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  48. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  49. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  51. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  52. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  53. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  56. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  58. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  59. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +2 -2
  60. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  61. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  62. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  63. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -2
  64. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  65. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  66. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  67. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  68. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  69. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  70. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  71. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_cli.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  79. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  84. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  85. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  86. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  87. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  88. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  89. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  90. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  91. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  92. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  93. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  94. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  95. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  96. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  97. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  98. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  99. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  100. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  101. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  102. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  103. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  105. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  106. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  107. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  108. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  109. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  110. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  112. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  113. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  114. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  115. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  116. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  118. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  119. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  120. metaflow-stubs/plugins/package_cli.pyi +2 -2
  121. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  122. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  123. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  124. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  126. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  127. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  128. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  129. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  130. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  131. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  132. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  133. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  134. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  135. metaflow-stubs/plugins/tag_cli.pyi +2 -2
  136. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  137. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  138. metaflow-stubs/procpoll.pyi +2 -2
  139. metaflow-stubs/pylint_wrapper.pyi +2 -2
  140. metaflow-stubs/runner/__init__.pyi +3 -3
  141. metaflow-stubs/runner/deployer.pyi +5 -5
  142. metaflow-stubs/runner/deployer_impl.pyi +3 -2
  143. metaflow-stubs/runner/metaflow_runner.pyi +8 -4
  144. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  145. metaflow-stubs/runner/nbrun.pyi +2 -2
  146. metaflow-stubs/runner/subprocess_manager.pyi +7 -3
  147. metaflow-stubs/runner/utils.pyi +108 -9
  148. metaflow-stubs/system/__init__.pyi +2 -2
  149. metaflow-stubs/system/system_logger.pyi +3 -3
  150. metaflow-stubs/system/system_monitor.pyi +2 -2
  151. metaflow-stubs/tagging_util.pyi +2 -2
  152. metaflow-stubs/tuple_util.pyi +2 -2
  153. metaflow-stubs/version.pyi +2 -2
  154. {metaflow_stubs-2.12.32.dist-info → metaflow_stubs-2.12.34.dist-info}/METADATA +2 -2
  155. metaflow_stubs-2.12.34.dist-info/RECORD +158 -0
  156. metaflow_stubs-2.12.32.dist-info/RECORD +0 -158
  157. {metaflow_stubs-2.12.32.dist-info → metaflow_stubs-2.12.34.dist-info}/WHEEL +0 -0
  158. {metaflow_stubs-2.12.32.dist-info → metaflow_stubs-2.12.34.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.32 #
4
- # Generated on 2024-11-26T19:51:56.545525 #
3
+ # MF version: 2.12.34 #
4
+ # Generated on 2024-12-04T14:12:34.722470 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -156,12 +156,6 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
156
156
  """
157
157
  ...
158
158
 
159
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
160
- """
161
- Specifies that this step should execute on Kubernetes.
162
- """
163
- ...
164
-
165
159
  @typing.overload
166
160
  def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
167
161
  """
@@ -184,90 +178,97 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
184
178
  ...
185
179
 
186
180
  @typing.overload
187
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
181
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
188
182
  """
189
- Specifies a timeout for your step.
190
-
191
- This decorator is useful if this step may hang indefinitely.
183
+ Specifies the number of times the task corresponding
184
+ to a step needs to be retried.
192
185
 
193
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
194
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
195
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
186
+ This decorator is useful for handling transient errors, such as networking issues.
187
+ If your task contains operations that can't be retried safely, e.g. database updates,
188
+ it is advisable to annotate it with `@retry(times=0)`.
196
189
 
197
- Note that all the values specified in parameters are added together so if you specify
198
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
190
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
191
+ decorator will execute a no-op task after all retries have been exhausted,
192
+ ensuring that the flow execution can continue.
199
193
  """
200
194
  ...
201
195
 
202
196
  @typing.overload
203
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
197
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
204
198
  ...
205
199
 
206
200
  @typing.overload
207
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
201
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
208
202
  ...
209
203
 
210
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
204
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
211
205
  """
212
- Specifies a timeout for your step.
206
+ Specifies the number of times the task corresponding
207
+ to a step needs to be retried.
213
208
 
214
- This decorator is useful if this step may hang indefinitely.
209
+ This decorator is useful for handling transient errors, such as networking issues.
210
+ If your task contains operations that can't be retried safely, e.g. database updates,
211
+ it is advisable to annotate it with `@retry(times=0)`.
215
212
 
216
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
217
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
218
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
213
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
214
+ decorator will execute a no-op task after all retries have been exhausted,
215
+ ensuring that the flow execution can continue.
216
+ """
217
+ ...
218
+
219
+ @typing.overload
220
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
221
+ """
222
+ Specifies that the step will success under all circumstances.
219
223
 
220
- Note that all the values specified in parameters are added together so if you specify
221
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
224
+ The decorator will create an optional artifact, specified by `var`, which
225
+ contains the exception raised. You can use it to detect the presence
226
+ of errors, indicating that all happy-path artifacts produced by the step
227
+ are missing.
222
228
  """
223
229
  ...
224
230
 
225
231
  @typing.overload
226
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
232
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
233
+ ...
234
+
235
+ @typing.overload
236
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
237
+ ...
238
+
239
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
227
240
  """
228
- Specifies the resources needed when executing this step.
241
+ Specifies that the step will success under all circumstances.
229
242
 
230
- Use `@resources` to specify the resource requirements
231
- independently of the specific compute layer (`@batch`, `@kubernetes`).
243
+ The decorator will create an optional artifact, specified by `var`, which
244
+ contains the exception raised. You can use it to detect the presence
245
+ of errors, indicating that all happy-path artifacts produced by the step
246
+ are missing.
247
+ """
248
+ ...
249
+
250
+ @typing.overload
251
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
252
+ """
253
+ Creates a human-readable report, a Metaflow Card, after this step completes.
232
254
 
233
- You can choose the compute layer on the command line by executing e.g.
234
- ```
235
- python myflow.py run --with batch
236
- ```
237
- or
238
- ```
239
- python myflow.py run --with kubernetes
240
- ```
241
- which executes the flow on the desired system using the
242
- requirements specified in `@resources`.
255
+ Note that you may add multiple `@card` decorators in a step with different parameters.
243
256
  """
244
257
  ...
245
258
 
246
259
  @typing.overload
247
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
260
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
248
261
  ...
249
262
 
250
263
  @typing.overload
251
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
264
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
252
265
  ...
253
266
 
254
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
267
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
255
268
  """
256
- Specifies the resources needed when executing this step.
257
-
258
- Use `@resources` to specify the resource requirements
259
- independently of the specific compute layer (`@batch`, `@kubernetes`).
269
+ Creates a human-readable report, a Metaflow Card, after this step completes.
260
270
 
261
- You can choose the compute layer on the command line by executing e.g.
262
- ```
263
- python myflow.py run --with batch
264
- ```
265
- or
266
- ```
267
- python myflow.py run --with kubernetes
268
- ```
269
- which executes the flow on the desired system using the
270
- requirements specified in `@resources`.
271
+ Note that you may add multiple `@card` decorators in a step with different parameters.
271
272
  """
272
273
  ...
273
274
 
@@ -316,189 +317,258 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
316
317
  ...
317
318
 
318
319
  @typing.overload
319
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
320
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
320
321
  """
321
- Specifies that the step will success under all circumstances.
322
+ Specifies the resources needed when executing this step.
322
323
 
323
- The decorator will create an optional artifact, specified by `var`, which
324
- contains the exception raised. You can use it to detect the presence
325
- of errors, indicating that all happy-path artifacts produced by the step
326
- are missing.
324
+ Use `@resources` to specify the resource requirements
325
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
326
+
327
+ You can choose the compute layer on the command line by executing e.g.
328
+ ```
329
+ python myflow.py run --with batch
330
+ ```
331
+ or
332
+ ```
333
+ python myflow.py run --with kubernetes
334
+ ```
335
+ which executes the flow on the desired system using the
336
+ requirements specified in `@resources`.
327
337
  """
328
338
  ...
329
339
 
330
340
  @typing.overload
331
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
341
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
332
342
  ...
333
343
 
334
344
  @typing.overload
335
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
345
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
336
346
  ...
337
347
 
338
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
348
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
339
349
  """
340
- Specifies that the step will success under all circumstances.
350
+ Specifies the resources needed when executing this step.
341
351
 
342
- The decorator will create an optional artifact, specified by `var`, which
343
- contains the exception raised. You can use it to detect the presence
344
- of errors, indicating that all happy-path artifacts produced by the step
345
- are missing.
346
- """
347
- ...
348
-
349
- @typing.overload
350
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
351
- """
352
- Specifies the Conda environment for the step.
352
+ Use `@resources` to specify the resource requirements
353
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
353
354
 
354
- Information in this decorator will augment any
355
- attributes set in the `@conda_base` flow-level decorator. Hence,
356
- you can use `@conda_base` to set packages required by all
357
- steps and use `@conda` to specify step-specific overrides.
355
+ You can choose the compute layer on the command line by executing e.g.
356
+ ```
357
+ python myflow.py run --with batch
358
+ ```
359
+ or
360
+ ```
361
+ python myflow.py run --with kubernetes
362
+ ```
363
+ which executes the flow on the desired system using the
364
+ requirements specified in `@resources`.
358
365
  """
359
366
  ...
360
367
 
361
- @typing.overload
362
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
363
- ...
364
-
365
- @typing.overload
366
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
367
- ...
368
-
369
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
368
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
370
369
  """
371
- Specifies the Conda environment for the step.
372
-
373
- Information in this decorator will augment any
374
- attributes set in the `@conda_base` flow-level decorator. Hence,
375
- you can use `@conda_base` to set packages required by all
376
- steps and use `@conda` to specify step-specific overrides.
370
+ Specifies that this step should execute on Kubernetes.
377
371
  """
378
372
  ...
379
373
 
380
374
  @typing.overload
381
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
375
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
382
376
  """
383
- Specifies the number of times the task corresponding
384
- to a step needs to be retried.
377
+ Specifies a timeout for your step.
385
378
 
386
- This decorator is useful for handling transient errors, such as networking issues.
387
- If your task contains operations that can't be retried safely, e.g. database updates,
388
- it is advisable to annotate it with `@retry(times=0)`.
379
+ This decorator is useful if this step may hang indefinitely.
389
380
 
390
- This can be used in conjunction with the `@catch` decorator. The `@catch`
391
- decorator will execute a no-op task after all retries have been exhausted,
392
- ensuring that the flow execution can continue.
381
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
382
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
383
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
384
+
385
+ Note that all the values specified in parameters are added together so if you specify
386
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
393
387
  """
394
388
  ...
395
389
 
396
390
  @typing.overload
397
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
391
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
398
392
  ...
399
393
 
400
394
  @typing.overload
401
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
395
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
402
396
  ...
403
397
 
404
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
398
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
405
399
  """
406
- Specifies the number of times the task corresponding
407
- to a step needs to be retried.
400
+ Specifies a timeout for your step.
408
401
 
409
- This decorator is useful for handling transient errors, such as networking issues.
410
- If your task contains operations that can't be retried safely, e.g. database updates,
411
- it is advisable to annotate it with `@retry(times=0)`.
402
+ This decorator is useful if this step may hang indefinitely.
412
403
 
413
- This can be used in conjunction with the `@catch` decorator. The `@catch`
414
- decorator will execute a no-op task after all retries have been exhausted,
415
- ensuring that the flow execution can continue.
404
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
405
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
406
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
407
+
408
+ Note that all the values specified in parameters are added together so if you specify
409
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
416
410
  """
417
411
  ...
418
412
 
419
413
  @typing.overload
420
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
414
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
421
415
  """
422
- Creates a human-readable report, a Metaflow Card, after this step completes.
416
+ Specifies the PyPI packages for the step.
423
417
 
424
- Note that you may add multiple `@card` decorators in a step with different parameters.
418
+ Information in this decorator will augment any
419
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
420
+ you can use `@pypi_base` to set packages required by all
421
+ steps and use `@pypi` to specify step-specific overrides.
425
422
  """
426
423
  ...
427
424
 
428
425
  @typing.overload
429
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
426
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
430
427
  ...
431
428
 
432
429
  @typing.overload
433
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
430
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
434
431
  ...
435
432
 
436
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
433
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
437
434
  """
438
- Creates a human-readable report, a Metaflow Card, after this step completes.
435
+ Specifies the PyPI packages for the step.
439
436
 
440
- Note that you may add multiple `@card` decorators in a step with different parameters.
437
+ Information in this decorator will augment any
438
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
439
+ you can use `@pypi_base` to set packages required by all
440
+ steps and use `@pypi` to specify step-specific overrides.
441
441
  """
442
442
  ...
443
443
 
444
444
  @typing.overload
445
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
445
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
446
446
  """
447
- Specifies the PyPI packages for the step.
447
+ Specifies the Conda environment for the step.
448
448
 
449
449
  Information in this decorator will augment any
450
- attributes set in the `@pyi_base` flow-level decorator. Hence,
451
- you can use `@pypi_base` to set packages required by all
452
- steps and use `@pypi` to specify step-specific overrides.
450
+ attributes set in the `@conda_base` flow-level decorator. Hence,
451
+ you can use `@conda_base` to set packages required by all
452
+ steps and use `@conda` to specify step-specific overrides.
453
453
  """
454
454
  ...
455
455
 
456
456
  @typing.overload
457
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
457
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
458
458
  ...
459
459
 
460
460
  @typing.overload
461
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
461
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
462
462
  ...
463
463
 
464
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
464
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
465
465
  """
466
- Specifies the PyPI packages for the step.
466
+ Specifies the Conda environment for the step.
467
467
 
468
468
  Information in this decorator will augment any
469
- attributes set in the `@pyi_base` flow-level decorator. Hence,
470
- you can use `@pypi_base` to set packages required by all
471
- steps and use `@pypi` to specify step-specific overrides.
469
+ attributes set in the `@conda_base` flow-level decorator. Hence,
470
+ you can use `@conda_base` to set packages required by all
471
+ steps and use `@conda` to specify step-specific overrides.
472
472
  """
473
473
  ...
474
474
 
475
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
475
+ @typing.overload
476
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
476
477
  """
477
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
478
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
478
+ Specifies the times when the flow should be run when running on a
479
+ production scheduler.
479
480
  """
480
481
  ...
481
482
 
482
483
  @typing.overload
483
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
484
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
485
+ ...
486
+
487
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
484
488
  """
485
- Specifies the Conda environment for all steps of the flow.
489
+ Specifies the times when the flow should be run when running on a
490
+ production scheduler.
491
+ """
492
+ ...
493
+
494
+ @typing.overload
495
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
496
+ """
497
+ Specifies the flow(s) that this flow depends on.
486
498
 
487
- Use `@conda_base` to set common libraries required by all
488
- steps and use `@conda` to specify step-specific additions.
499
+ ```
500
+ @trigger_on_finish(flow='FooFlow')
501
+ ```
502
+ or
503
+ ```
504
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
505
+ ```
506
+ This decorator respects the @project decorator and triggers the flow
507
+ when upstream runs within the same namespace complete successfully
508
+
509
+ Additionally, you can specify project aware upstream flow dependencies
510
+ by specifying the fully qualified project_flow_name.
511
+ ```
512
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
513
+ ```
514
+ or
515
+ ```
516
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
517
+ ```
518
+
519
+ You can also specify just the project or project branch (other values will be
520
+ inferred from the current project or project branch):
521
+ ```
522
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
523
+ ```
524
+
525
+ Note that `branch` is typically one of:
526
+ - `prod`
527
+ - `user.bob`
528
+ - `test.my_experiment`
529
+ - `prod.staging`
489
530
  """
490
531
  ...
491
532
 
492
533
  @typing.overload
493
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
534
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
494
535
  ...
495
536
 
496
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
537
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
497
538
  """
498
- Specifies the Conda environment for all steps of the flow.
539
+ Specifies the flow(s) that this flow depends on.
499
540
 
500
- Use `@conda_base` to set common libraries required by all
501
- steps and use `@conda` to specify step-specific additions.
541
+ ```
542
+ @trigger_on_finish(flow='FooFlow')
543
+ ```
544
+ or
545
+ ```
546
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
547
+ ```
548
+ This decorator respects the @project decorator and triggers the flow
549
+ when upstream runs within the same namespace complete successfully
550
+
551
+ Additionally, you can specify project aware upstream flow dependencies
552
+ by specifying the fully qualified project_flow_name.
553
+ ```
554
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
555
+ ```
556
+ or
557
+ ```
558
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
559
+ ```
560
+
561
+ You can also specify just the project or project branch (other values will be
562
+ inferred from the current project or project branch):
563
+ ```
564
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
565
+ ```
566
+
567
+ Note that `branch` is typically one of:
568
+ - `prod`
569
+ - `user.bob`
570
+ - `test.my_experiment`
571
+ - `prod.staging`
502
572
  """
503
573
  ...
504
574
 
@@ -511,6 +581,23 @@ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typ
511
581
  """
512
582
  ...
513
583
 
584
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
585
+ """
586
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
587
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
588
+ """
589
+ ...
590
+
591
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
592
+ """
593
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
594
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
595
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
596
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
597
+ starts only after all sensors finish.
598
+ """
599
+ ...
600
+
514
601
  @typing.overload
515
602
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
516
603
  """
@@ -608,112 +695,25 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
608
695
  ...
609
696
 
610
697
  @typing.overload
611
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
698
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
612
699
  """
613
- Specifies the flow(s) that this flow depends on.
614
-
615
- ```
616
- @trigger_on_finish(flow='FooFlow')
617
- ```
618
- or
619
- ```
620
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
621
- ```
622
- This decorator respects the @project decorator and triggers the flow
623
- when upstream runs within the same namespace complete successfully
624
-
625
- Additionally, you can specify project aware upstream flow dependencies
626
- by specifying the fully qualified project_flow_name.
627
- ```
628
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
629
- ```
630
- or
631
- ```
632
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
633
- ```
634
-
635
- You can also specify just the project or project branch (other values will be
636
- inferred from the current project or project branch):
637
- ```
638
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
639
- ```
700
+ Specifies the Conda environment for all steps of the flow.
640
701
 
641
- Note that `branch` is typically one of:
642
- - `prod`
643
- - `user.bob`
644
- - `test.my_experiment`
645
- - `prod.staging`
702
+ Use `@conda_base` to set common libraries required by all
703
+ steps and use `@conda` to specify step-specific additions.
646
704
  """
647
705
  ...
648
706
 
649
707
  @typing.overload
650
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
708
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
651
709
  ...
652
710
 
653
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
711
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
654
712
  """
655
- Specifies the flow(s) that this flow depends on.
656
-
657
- ```
658
- @trigger_on_finish(flow='FooFlow')
659
- ```
660
- or
661
- ```
662
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
663
- ```
664
- This decorator respects the @project decorator and triggers the flow
665
- when upstream runs within the same namespace complete successfully
666
-
667
- Additionally, you can specify project aware upstream flow dependencies
668
- by specifying the fully qualified project_flow_name.
669
- ```
670
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
671
- ```
672
- or
673
- ```
674
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
675
- ```
676
-
677
- You can also specify just the project or project branch (other values will be
678
- inferred from the current project or project branch):
679
- ```
680
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
681
- ```
713
+ Specifies the Conda environment for all steps of the flow.
682
714
 
683
- Note that `branch` is typically one of:
684
- - `prod`
685
- - `user.bob`
686
- - `test.my_experiment`
687
- - `prod.staging`
688
- """
689
- ...
690
-
691
- @typing.overload
692
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
693
- """
694
- Specifies the times when the flow should be run when running on a
695
- production scheduler.
696
- """
697
- ...
698
-
699
- @typing.overload
700
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
701
- ...
702
-
703
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
704
- """
705
- Specifies the times when the flow should be run when running on a
706
- production scheduler.
707
- """
708
- ...
709
-
710
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
711
- """
712
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
713
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
714
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
715
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
716
- starts only after all sensors finish.
715
+ Use `@conda_base` to set common libraries required by all
716
+ steps and use `@conda` to specify step-specific additions.
717
717
  """
718
718
  ...
719
719