metaflow-stubs 2.12.33__py2.py3-none-any.whl → 2.12.34__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. metaflow-stubs/__init__.pyi +250 -250
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +2 -2
  5. metaflow-stubs/client/core.pyi +3 -3
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +2 -2
  13. metaflow-stubs/info_file.pyi +2 -2
  14. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  16. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  17. metaflow-stubs/metadata_provider/util.pyi +2 -2
  18. metaflow-stubs/metaflow_config.pyi +2 -2
  19. metaflow-stubs/metaflow_current.pyi +24 -24
  20. metaflow-stubs/mflog/__init__.pyi +2 -2
  21. metaflow-stubs/mflog/mflog.pyi +2 -2
  22. metaflow-stubs/multicore_utils.pyi +11 -11
  23. metaflow-stubs/parameters.pyi +2 -2
  24. metaflow-stubs/plugins/__init__.pyi +13 -13
  25. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/airflow.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  31. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  33. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  34. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  35. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  37. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  38. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  39. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +2 -2
  40. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  41. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  42. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -2
  43. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  46. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  47. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  48. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  49. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  51. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  52. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  53. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  56. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  58. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  59. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +2 -2
  60. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  61. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  62. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  63. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -2
  64. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  65. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  66. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  67. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  68. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  69. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  70. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  71. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_cli.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  74. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  79. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  84. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  85. metaflow-stubs/plugins/cards/card_modules/components.pyi +2 -2
  86. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  87. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  88. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  89. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  90. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  91. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  92. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  93. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  94. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  95. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  96. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  97. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  98. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  99. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  100. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  101. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  102. metaflow-stubs/plugins/events_decorator.pyi +12 -2
  103. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  105. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  106. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  107. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  108. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  109. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  110. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  112. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  113. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  114. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  115. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  116. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  118. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  119. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  120. metaflow-stubs/plugins/package_cli.pyi +2 -2
  121. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  122. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  123. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  124. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  126. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  127. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  128. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  129. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  130. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  131. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  132. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  133. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  134. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  135. metaflow-stubs/plugins/tag_cli.pyi +2 -2
  136. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  137. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  138. metaflow-stubs/procpoll.pyi +2 -2
  139. metaflow-stubs/pylint_wrapper.pyi +2 -2
  140. metaflow-stubs/runner/__init__.pyi +3 -3
  141. metaflow-stubs/runner/deployer.pyi +4 -4
  142. metaflow-stubs/runner/deployer_impl.pyi +3 -2
  143. metaflow-stubs/runner/metaflow_runner.pyi +8 -4
  144. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  145. metaflow-stubs/runner/nbrun.pyi +2 -2
  146. metaflow-stubs/runner/subprocess_manager.pyi +7 -3
  147. metaflow-stubs/runner/utils.pyi +108 -9
  148. metaflow-stubs/system/__init__.pyi +2 -2
  149. metaflow-stubs/system/system_logger.pyi +3 -3
  150. metaflow-stubs/system/system_monitor.pyi +2 -2
  151. metaflow-stubs/tagging_util.pyi +2 -2
  152. metaflow-stubs/tuple_util.pyi +2 -2
  153. metaflow-stubs/version.pyi +2 -2
  154. {metaflow_stubs-2.12.33.dist-info → metaflow_stubs-2.12.34.dist-info}/METADATA +2 -2
  155. metaflow_stubs-2.12.34.dist-info/RECORD +158 -0
  156. metaflow_stubs-2.12.33.dist-info/RECORD +0 -158
  157. {metaflow_stubs-2.12.33.dist-info → metaflow_stubs-2.12.34.dist-info}/WHEEL +0 -0
  158. {metaflow_stubs-2.12.33.dist-info → metaflow_stubs-2.12.34.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.33 #
4
- # Generated on 2024-11-27T23:43:46.597947 #
3
+ # MF version: 2.12.34 #
4
+ # Generated on 2024-12-04T14:12:34.722470 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -29,8 +29,8 @@ from .flowspec import FlowSpec as FlowSpec
29
29
  from .parameters import Parameter as Parameter
30
30
  from .parameters import JSONTypeClass as JSONTypeClass
31
31
  from .parameters import JSONType as JSONType
32
- from . import events as events
33
32
  from . import tuple_util as tuple_util
33
+ from . import events as events
34
34
  from . import runner as runner
35
35
  from . import plugins as plugins
36
36
  from .plugins.datatools.s3.s3 import S3 as S3
@@ -138,66 +138,81 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
138
138
  ...
139
139
 
140
140
  @typing.overload
141
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
141
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
142
142
  """
143
- Creates a human-readable report, a Metaflow Card, after this step completes.
144
-
145
- Note that you may add multiple `@card` decorators in a step with different parameters.
143
+ Decorator prototype for all step decorators. This function gets specialized
144
+ and imported for all decorators types by _import_plugin_decorators().
146
145
  """
147
146
  ...
148
147
 
149
148
  @typing.overload
150
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
149
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
150
+ ...
151
+
152
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
153
+ """
154
+ Decorator prototype for all step decorators. This function gets specialized
155
+ and imported for all decorators types by _import_plugin_decorators().
156
+ """
151
157
  ...
152
158
 
153
159
  @typing.overload
154
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
160
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
161
+ """
162
+ Specifies environment variables to be set prior to the execution of a step.
163
+ """
155
164
  ...
156
165
 
157
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
166
+ @typing.overload
167
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
168
+ ...
169
+
170
+ @typing.overload
171
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
172
+ ...
173
+
174
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
158
175
  """
159
- Creates a human-readable report, a Metaflow Card, after this step completes.
160
-
161
- Note that you may add multiple `@card` decorators in a step with different parameters.
176
+ Specifies environment variables to be set prior to the execution of a step.
162
177
  """
163
178
  ...
164
179
 
165
180
  @typing.overload
166
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
181
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
167
182
  """
168
- Specifies a timeout for your step.
169
-
170
- This decorator is useful if this step may hang indefinitely.
183
+ Specifies the number of times the task corresponding
184
+ to a step needs to be retried.
171
185
 
172
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
173
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
174
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
186
+ This decorator is useful for handling transient errors, such as networking issues.
187
+ If your task contains operations that can't be retried safely, e.g. database updates,
188
+ it is advisable to annotate it with `@retry(times=0)`.
175
189
 
176
- Note that all the values specified in parameters are added together so if you specify
177
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
190
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
191
+ decorator will execute a no-op task after all retries have been exhausted,
192
+ ensuring that the flow execution can continue.
178
193
  """
179
194
  ...
180
195
 
181
196
  @typing.overload
182
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
197
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
183
198
  ...
184
199
 
185
200
  @typing.overload
186
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
201
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
187
202
  ...
188
203
 
189
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
204
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
190
205
  """
191
- Specifies a timeout for your step.
192
-
193
- This decorator is useful if this step may hang indefinitely.
206
+ Specifies the number of times the task corresponding
207
+ to a step needs to be retried.
194
208
 
195
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
196
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
197
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
209
+ This decorator is useful for handling transient errors, such as networking issues.
210
+ If your task contains operations that can't be retried safely, e.g. database updates,
211
+ it is advisable to annotate it with `@retry(times=0)`.
198
212
 
199
- Note that all the values specified in parameters are added together so if you specify
200
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
213
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
214
+ decorator will execute a no-op task after all retries have been exhausted,
215
+ ensuring that the flow execution can continue.
201
216
  """
202
217
  ...
203
218
 
@@ -233,51 +248,50 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
233
248
  ...
234
249
 
235
250
  @typing.overload
236
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
251
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
237
252
  """
238
- Specifies the resources needed when executing this step.
239
-
240
- Use `@resources` to specify the resource requirements
241
- independently of the specific compute layer (`@batch`, `@kubernetes`).
253
+ Creates a human-readable report, a Metaflow Card, after this step completes.
242
254
 
243
- You can choose the compute layer on the command line by executing e.g.
244
- ```
245
- python myflow.py run --with batch
246
- ```
247
- or
248
- ```
249
- python myflow.py run --with kubernetes
250
- ```
251
- which executes the flow on the desired system using the
252
- requirements specified in `@resources`.
255
+ Note that you may add multiple `@card` decorators in a step with different parameters.
253
256
  """
254
257
  ...
255
258
 
256
259
  @typing.overload
257
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
260
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
258
261
  ...
259
262
 
260
263
  @typing.overload
261
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
264
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
262
265
  ...
263
266
 
264
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
267
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
265
268
  """
266
- Specifies the resources needed when executing this step.
267
-
268
- Use `@resources` to specify the resource requirements
269
- independently of the specific compute layer (`@batch`, `@kubernetes`).
269
+ Creates a human-readable report, a Metaflow Card, after this step completes.
270
270
 
271
- You can choose the compute layer on the command line by executing e.g.
272
- ```
273
- python myflow.py run --with batch
274
- ```
275
- or
276
- ```
277
- python myflow.py run --with kubernetes
278
- ```
279
- which executes the flow on the desired system using the
280
- requirements specified in `@resources`.
271
+ Note that you may add multiple `@card` decorators in a step with different parameters.
272
+ """
273
+ ...
274
+
275
+ @typing.overload
276
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
277
+ """
278
+ Specifies secrets to be retrieved and injected as environment variables prior to
279
+ the execution of a step.
280
+ """
281
+ ...
282
+
283
+ @typing.overload
284
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
285
+ ...
286
+
287
+ @typing.overload
288
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
289
+ ...
290
+
291
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
292
+ """
293
+ Specifies secrets to be retrieved and injected as environment variables prior to
294
+ the execution of a step.
281
295
  """
282
296
  ...
283
297
 
@@ -303,23 +317,51 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
303
317
  ...
304
318
 
305
319
  @typing.overload
306
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
320
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
307
321
  """
308
- Specifies environment variables to be set prior to the execution of a step.
322
+ Specifies the resources needed when executing this step.
323
+
324
+ Use `@resources` to specify the resource requirements
325
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
326
+
327
+ You can choose the compute layer on the command line by executing e.g.
328
+ ```
329
+ python myflow.py run --with batch
330
+ ```
331
+ or
332
+ ```
333
+ python myflow.py run --with kubernetes
334
+ ```
335
+ which executes the flow on the desired system using the
336
+ requirements specified in `@resources`.
309
337
  """
310
338
  ...
311
339
 
312
340
  @typing.overload
313
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
341
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
314
342
  ...
315
343
 
316
344
  @typing.overload
317
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
345
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
318
346
  ...
319
347
 
320
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
348
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
321
349
  """
322
- Specifies environment variables to be set prior to the execution of a step.
350
+ Specifies the resources needed when executing this step.
351
+
352
+ Use `@resources` to specify the resource requirements
353
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
354
+
355
+ You can choose the compute layer on the command line by executing e.g.
356
+ ```
357
+ python myflow.py run --with batch
358
+ ```
359
+ or
360
+ ```
361
+ python myflow.py run --with kubernetes
362
+ ```
363
+ which executes the flow on the desired system using the
364
+ requirements specified in `@resources`.
323
365
  """
324
366
  ...
325
367
 
@@ -330,25 +372,41 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
330
372
  ...
331
373
 
332
374
  @typing.overload
333
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
375
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
334
376
  """
335
- Specifies secrets to be retrieved and injected as environment variables prior to
336
- the execution of a step.
377
+ Specifies a timeout for your step.
378
+
379
+ This decorator is useful if this step may hang indefinitely.
380
+
381
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
382
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
383
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
384
+
385
+ Note that all the values specified in parameters are added together so if you specify
386
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
337
387
  """
338
388
  ...
339
389
 
340
390
  @typing.overload
341
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
391
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
342
392
  ...
343
393
 
344
394
  @typing.overload
345
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
395
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
346
396
  ...
347
397
 
348
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
398
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
349
399
  """
350
- Specifies secrets to be retrieved and injected as environment variables prior to
351
- the execution of a step.
400
+ Specifies a timeout for your step.
401
+
402
+ This decorator is useful if this step may hang indefinitely.
403
+
404
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
405
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
406
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
407
+
408
+ Note that all the values specified in parameters are added together so if you specify
409
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
352
410
  """
353
411
  ...
354
412
 
@@ -384,46 +442,26 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
384
442
  ...
385
443
 
386
444
  @typing.overload
387
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
445
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
388
446
  """
389
- Specifies the number of times the task corresponding
390
- to a step needs to be retried.
391
-
392
- This decorator is useful for handling transient errors, such as networking issues.
393
- If your task contains operations that can't be retried safely, e.g. database updates,
394
- it is advisable to annotate it with `@retry(times=0)`.
447
+ Specifies the Conda environment for the step.
395
448
 
396
- This can be used in conjunction with the `@catch` decorator. The `@catch`
397
- decorator will execute a no-op task after all retries have been exhausted,
398
- ensuring that the flow execution can continue.
449
+ Information in this decorator will augment any
450
+ attributes set in the `@conda_base` flow-level decorator. Hence,
451
+ you can use `@conda_base` to set packages required by all
452
+ steps and use `@conda` to specify step-specific overrides.
399
453
  """
400
454
  ...
401
455
 
402
456
  @typing.overload
403
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
457
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
404
458
  ...
405
459
 
406
460
  @typing.overload
407
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
408
- ...
409
-
410
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
411
- """
412
- Specifies the number of times the task corresponding
413
- to a step needs to be retried.
414
-
415
- This decorator is useful for handling transient errors, such as networking issues.
416
- If your task contains operations that can't be retried safely, e.g. database updates,
417
- it is advisable to annotate it with `@retry(times=0)`.
418
-
419
- This can be used in conjunction with the `@catch` decorator. The `@catch`
420
- decorator will execute a no-op task after all retries have been exhausted,
421
- ensuring that the flow execution can continue.
422
- """
461
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
423
462
  ...
424
463
 
425
- @typing.overload
426
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
464
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
427
465
  """
428
466
  Specifies the Conda environment for the step.
429
467
 
@@ -435,40 +473,128 @@ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, s
435
473
  ...
436
474
 
437
475
  @typing.overload
438
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
476
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
477
+ """
478
+ Specifies the times when the flow should be run when running on a
479
+ production scheduler.
480
+ """
439
481
  ...
440
482
 
441
483
  @typing.overload
442
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
484
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
443
485
  ...
444
486
 
445
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
487
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
446
488
  """
447
- Specifies the Conda environment for the step.
448
-
449
- Information in this decorator will augment any
450
- attributes set in the `@conda_base` flow-level decorator. Hence,
451
- you can use `@conda_base` to set packages required by all
452
- steps and use `@conda` to specify step-specific overrides.
489
+ Specifies the times when the flow should be run when running on a
490
+ production scheduler.
453
491
  """
454
492
  ...
455
493
 
456
494
  @typing.overload
457
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
495
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
458
496
  """
459
- Decorator prototype for all step decorators. This function gets specialized
460
- and imported for all decorators types by _import_plugin_decorators().
497
+ Specifies the flow(s) that this flow depends on.
498
+
499
+ ```
500
+ @trigger_on_finish(flow='FooFlow')
501
+ ```
502
+ or
503
+ ```
504
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
505
+ ```
506
+ This decorator respects the @project decorator and triggers the flow
507
+ when upstream runs within the same namespace complete successfully
508
+
509
+ Additionally, you can specify project aware upstream flow dependencies
510
+ by specifying the fully qualified project_flow_name.
511
+ ```
512
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
513
+ ```
514
+ or
515
+ ```
516
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
517
+ ```
518
+
519
+ You can also specify just the project or project branch (other values will be
520
+ inferred from the current project or project branch):
521
+ ```
522
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
523
+ ```
524
+
525
+ Note that `branch` is typically one of:
526
+ - `prod`
527
+ - `user.bob`
528
+ - `test.my_experiment`
529
+ - `prod.staging`
461
530
  """
462
531
  ...
463
532
 
464
533
  @typing.overload
465
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
534
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
466
535
  ...
467
536
 
468
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
537
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
469
538
  """
470
- Decorator prototype for all step decorators. This function gets specialized
471
- and imported for all decorators types by _import_plugin_decorators().
539
+ Specifies the flow(s) that this flow depends on.
540
+
541
+ ```
542
+ @trigger_on_finish(flow='FooFlow')
543
+ ```
544
+ or
545
+ ```
546
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
547
+ ```
548
+ This decorator respects the @project decorator and triggers the flow
549
+ when upstream runs within the same namespace complete successfully
550
+
551
+ Additionally, you can specify project aware upstream flow dependencies
552
+ by specifying the fully qualified project_flow_name.
553
+ ```
554
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
555
+ ```
556
+ or
557
+ ```
558
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
559
+ ```
560
+
561
+ You can also specify just the project or project branch (other values will be
562
+ inferred from the current project or project branch):
563
+ ```
564
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
565
+ ```
566
+
567
+ Note that `branch` is typically one of:
568
+ - `prod`
569
+ - `user.bob`
570
+ - `test.my_experiment`
571
+ - `prod.staging`
572
+ """
573
+ ...
574
+
575
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
576
+ """
577
+ Specifies what flows belong to the same project.
578
+
579
+ A project-specific namespace is created for all flows that
580
+ use the same `@project(name)`.
581
+ """
582
+ ...
583
+
584
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
585
+ """
586
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
587
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
588
+ """
589
+ ...
590
+
591
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
592
+ """
593
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
594
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
595
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
596
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
597
+ starts only after all sensors finish.
472
598
  """
473
599
  ...
474
600
 
@@ -545,22 +671,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
545
671
  """
546
672
  ...
547
673
 
548
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
549
- """
550
- Specifies what flows belong to the same project.
551
-
552
- A project-specific namespace is created for all flows that
553
- use the same `@project(name)`.
554
- """
555
- ...
556
-
557
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
558
- """
559
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
560
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
561
- """
562
- ...
563
-
564
674
  @typing.overload
565
675
  def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
566
676
  """
@@ -584,16 +694,6 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
584
694
  """
585
695
  ...
586
696
 
587
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
588
- """
589
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
590
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
591
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
592
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
593
- starts only after all sensors finish.
594
- """
595
- ...
596
-
597
697
  @typing.overload
598
698
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
599
699
  """
@@ -617,103 +717,3 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
617
717
  """
618
718
  ...
619
719
 
620
- @typing.overload
621
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
622
- """
623
- Specifies the times when the flow should be run when running on a
624
- production scheduler.
625
- """
626
- ...
627
-
628
- @typing.overload
629
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
630
- ...
631
-
632
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
633
- """
634
- Specifies the times when the flow should be run when running on a
635
- production scheduler.
636
- """
637
- ...
638
-
639
- @typing.overload
640
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
641
- """
642
- Specifies the flow(s) that this flow depends on.
643
-
644
- ```
645
- @trigger_on_finish(flow='FooFlow')
646
- ```
647
- or
648
- ```
649
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
650
- ```
651
- This decorator respects the @project decorator and triggers the flow
652
- when upstream runs within the same namespace complete successfully
653
-
654
- Additionally, you can specify project aware upstream flow dependencies
655
- by specifying the fully qualified project_flow_name.
656
- ```
657
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
658
- ```
659
- or
660
- ```
661
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
662
- ```
663
-
664
- You can also specify just the project or project branch (other values will be
665
- inferred from the current project or project branch):
666
- ```
667
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
668
- ```
669
-
670
- Note that `branch` is typically one of:
671
- - `prod`
672
- - `user.bob`
673
- - `test.my_experiment`
674
- - `prod.staging`
675
- """
676
- ...
677
-
678
- @typing.overload
679
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
680
- ...
681
-
682
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
683
- """
684
- Specifies the flow(s) that this flow depends on.
685
-
686
- ```
687
- @trigger_on_finish(flow='FooFlow')
688
- ```
689
- or
690
- ```
691
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
692
- ```
693
- This decorator respects the @project decorator and triggers the flow
694
- when upstream runs within the same namespace complete successfully
695
-
696
- Additionally, you can specify project aware upstream flow dependencies
697
- by specifying the fully qualified project_flow_name.
698
- ```
699
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
700
- ```
701
- or
702
- ```
703
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
704
- ```
705
-
706
- You can also specify just the project or project branch (other values will be
707
- inferred from the current project or project branch):
708
- ```
709
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
710
- ```
711
-
712
- Note that `branch` is typically one of:
713
- - `prod`
714
- - `user.bob`
715
- - `test.my_experiment`
716
- - `prod.staging`
717
- """
718
- ...
719
-