metaflow-stubs 2.18.8__py2.py3-none-any.whl → 2.18.9__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of metaflow-stubs might be problematic. Click here for more details.

Files changed (167) hide show
  1. metaflow-stubs/__init__.pyi +570 -569
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +7 -7
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +6 -6
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +13 -6
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +23 -23
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/multicore_utils.pyi +2 -2
  23. metaflow-stubs/packaging_sys/__init__.pyi +6 -6
  24. metaflow-stubs/packaging_sys/backend.pyi +3 -3
  25. metaflow-stubs/packaging_sys/distribution_support.pyi +4 -4
  26. metaflow-stubs/packaging_sys/tar_backend.pyi +5 -5
  27. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  28. metaflow-stubs/packaging_sys/v1.pyi +4 -4
  29. metaflow-stubs/parameters.pyi +4 -4
  30. metaflow-stubs/plugins/__init__.pyi +13 -12
  31. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  33. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  34. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  35. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  37. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  38. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  39. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  41. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  42. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  43. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +2 -2
  44. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +5 -5
  45. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -4
  46. metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
  47. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  50. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  51. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  52. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  53. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  54. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  55. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  56. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  58. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  59. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  60. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  61. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  62. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  63. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  64. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  65. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  67. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  68. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  69. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  70. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  71. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  72. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  77. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  81. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  84. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  85. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  86. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  87. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  88. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/datatools/s3/s3.pyi +3 -3
  92. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  93. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  94. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  95. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  96. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  104. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  105. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  106. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  107. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  108. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  110. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  112. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  114. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  115. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  116. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/parsers.pyi +25 -0
  118. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  119. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  120. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  121. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  122. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  123. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  124. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  125. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  126. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  127. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  128. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  129. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  130. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  131. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  132. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  133. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  134. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  135. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  136. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  137. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  138. metaflow-stubs/plugins/uv/uv_environment.pyi +2 -2
  139. metaflow-stubs/pylint_wrapper.pyi +2 -2
  140. metaflow-stubs/runner/__init__.pyi +2 -2
  141. metaflow-stubs/runner/deployer.pyi +33 -33
  142. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  143. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  144. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  145. metaflow-stubs/runner/nbrun.pyi +2 -2
  146. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  147. metaflow-stubs/runner/utils.pyi +4 -4
  148. metaflow-stubs/system/__init__.pyi +2 -2
  149. metaflow-stubs/system/system_logger.pyi +2 -2
  150. metaflow-stubs/system/system_monitor.pyi +2 -2
  151. metaflow-stubs/tagging_util.pyi +2 -2
  152. metaflow-stubs/tuple_util.pyi +2 -2
  153. metaflow-stubs/user_configs/__init__.pyi +2 -2
  154. metaflow-stubs/user_configs/config_options.pyi +3 -3
  155. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  156. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  157. metaflow-stubs/user_decorators/common.pyi +2 -2
  158. metaflow-stubs/user_decorators/mutable_flow.pyi +6 -6
  159. metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
  160. metaflow-stubs/user_decorators/user_flow_decorator.pyi +4 -4
  161. metaflow-stubs/user_decorators/user_step_decorator.pyi +7 -7
  162. metaflow-stubs/version.pyi +2 -2
  163. {metaflow_stubs-2.18.8.dist-info → metaflow_stubs-2.18.9.dist-info}/METADATA +2 -2
  164. metaflow_stubs-2.18.9.dist-info/RECORD +167 -0
  165. metaflow_stubs-2.18.8.dist-info/RECORD +0 -166
  166. {metaflow_stubs-2.18.8.dist-info → metaflow_stubs-2.18.9.dist-info}/WHEEL +0 -0
  167. {metaflow_stubs-2.18.8.dist-info → metaflow_stubs-2.18.9.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.18.8 #
4
- # Generated on 2025-09-22T21:43:14.168437 #
3
+ # MF version: 2.18.9 #
4
+ # Generated on 2025-09-24T20:09:24.398546 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -40,16 +40,17 @@ from .user_decorators.user_step_decorator import StepMutator as StepMutator
40
40
  from .user_decorators.user_step_decorator import user_step_decorator as user_step_decorator
41
41
  from .user_decorators.user_flow_decorator import FlowMutator as FlowMutator
42
42
  from . import tuple_util as tuple_util
43
- from . import metaflow_git as metaflow_git
44
43
  from . import events as events
44
+ from . import metaflow_git as metaflow_git
45
45
  from . import runner as runner
46
46
  from . import plugins as plugins
47
47
  from .plugins.datatools.s3.s3 import S3 as S3
48
48
  from . import includefile as includefile
49
49
  from .includefile import IncludeFile as IncludeFile
50
+ from .plugins.parsers import yaml_parser as yaml_parser
51
+ from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
50
52
  from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
51
53
  from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
52
- from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
53
54
  from . import cards as cards
54
55
  from . import client as client
55
56
  from .client.core import namespace as namespace
@@ -152,119 +153,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
152
153
  """
153
154
  ...
154
155
 
155
- @typing.overload
156
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
157
- """
158
- Specifies the number of times the task corresponding
159
- to a step needs to be retried.
160
-
161
- This decorator is useful for handling transient errors, such as networking issues.
162
- If your task contains operations that can't be retried safely, e.g. database updates,
163
- it is advisable to annotate it with `@retry(times=0)`.
164
-
165
- This can be used in conjunction with the `@catch` decorator. The `@catch`
166
- decorator will execute a no-op task after all retries have been exhausted,
167
- ensuring that the flow execution can continue.
168
-
169
-
170
- Parameters
171
- ----------
172
- times : int, default 3
173
- Number of times to retry this task.
174
- minutes_between_retries : int, default 2
175
- Number of minutes between retries.
176
- """
177
- ...
178
-
179
- @typing.overload
180
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
181
- ...
182
-
183
- @typing.overload
184
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
185
- ...
186
-
187
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
188
- """
189
- Specifies the number of times the task corresponding
190
- to a step needs to be retried.
191
-
192
- This decorator is useful for handling transient errors, such as networking issues.
193
- If your task contains operations that can't be retried safely, e.g. database updates,
194
- it is advisable to annotate it with `@retry(times=0)`.
195
-
196
- This can be used in conjunction with the `@catch` decorator. The `@catch`
197
- decorator will execute a no-op task after all retries have been exhausted,
198
- ensuring that the flow execution can continue.
199
-
200
-
201
- Parameters
202
- ----------
203
- times : int, default 3
204
- Number of times to retry this task.
205
- minutes_between_retries : int, default 2
206
- Number of minutes between retries.
207
- """
208
- ...
209
-
210
- @typing.overload
211
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
212
- """
213
- Specifies secrets to be retrieved and injected as environment variables prior to
214
- the execution of a step.
215
-
216
-
217
- Parameters
218
- ----------
219
- sources : List[Union[str, Dict[str, Any]]], default: []
220
- List of secret specs, defining how the secrets are to be retrieved
221
- role : str, optional, default: None
222
- Role to use for fetching secrets
223
- """
224
- ...
225
-
226
- @typing.overload
227
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
228
- ...
229
-
230
- @typing.overload
231
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
232
- ...
233
-
234
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
235
- """
236
- Specifies secrets to be retrieved and injected as environment variables prior to
237
- the execution of a step.
238
-
239
-
240
- Parameters
241
- ----------
242
- sources : List[Union[str, Dict[str, Any]]], default: []
243
- List of secret specs, defining how the secrets are to be retrieved
244
- role : str, optional, default: None
245
- Role to use for fetching secrets
246
- """
247
- ...
248
-
249
- @typing.overload
250
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
251
- """
252
- Decorator prototype for all step decorators. This function gets specialized
253
- and imported for all decorators types by _import_plugin_decorators().
254
- """
255
- ...
256
-
257
- @typing.overload
258
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
259
- ...
260
-
261
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
262
- """
263
- Decorator prototype for all step decorators. This function gets specialized
264
- and imported for all decorators types by _import_plugin_decorators().
265
- """
266
- ...
267
-
268
156
  @typing.overload
269
157
  def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, aws_batch_tags: typing.Optional[typing.Dict[str, str]] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
270
158
  """
@@ -420,98 +308,6 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
420
308
  """
421
309
  ...
422
310
 
423
- @typing.overload
424
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
425
- """
426
- Specifies the Conda environment for the step.
427
-
428
- Information in this decorator will augment any
429
- attributes set in the `@conda_base` flow-level decorator. Hence,
430
- you can use `@conda_base` to set packages required by all
431
- steps and use `@conda` to specify step-specific overrides.
432
-
433
-
434
- Parameters
435
- ----------
436
- packages : Dict[str, str], default {}
437
- Packages to use for this step. The key is the name of the package
438
- and the value is the version to use.
439
- libraries : Dict[str, str], default {}
440
- Supported for backward compatibility. When used with packages, packages will take precedence.
441
- python : str, optional, default None
442
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
443
- that the version used will correspond to the version of the Python interpreter used to start the run.
444
- disabled : bool, default False
445
- If set to True, disables @conda.
446
- """
447
- ...
448
-
449
- @typing.overload
450
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
451
- ...
452
-
453
- @typing.overload
454
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
455
- ...
456
-
457
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
458
- """
459
- Specifies the Conda environment for the step.
460
-
461
- Information in this decorator will augment any
462
- attributes set in the `@conda_base` flow-level decorator. Hence,
463
- you can use `@conda_base` to set packages required by all
464
- steps and use `@conda` to specify step-specific overrides.
465
-
466
-
467
- Parameters
468
- ----------
469
- packages : Dict[str, str], default {}
470
- Packages to use for this step. The key is the name of the package
471
- and the value is the version to use.
472
- libraries : Dict[str, str], default {}
473
- Supported for backward compatibility. When used with packages, packages will take precedence.
474
- python : str, optional, default None
475
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
476
- that the version used will correspond to the version of the Python interpreter used to start the run.
477
- disabled : bool, default False
478
- If set to True, disables @conda.
479
- """
480
- ...
481
-
482
- @typing.overload
483
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
484
- """
485
- Specifies environment variables to be set prior to the execution of a step.
486
-
487
-
488
- Parameters
489
- ----------
490
- vars : Dict[str, str], default {}
491
- Dictionary of environment variables to set.
492
- """
493
- ...
494
-
495
- @typing.overload
496
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
497
- ...
498
-
499
- @typing.overload
500
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
501
- ...
502
-
503
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
504
- """
505
- Specifies environment variables to be set prior to the execution of a step.
506
-
507
-
508
- Parameters
509
- ----------
510
- vars : Dict[str, str], default {}
511
- Dictionary of environment variables to set.
512
- """
513
- ...
514
-
515
311
  @typing.overload
516
312
  def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
517
313
  """
@@ -572,62 +368,39 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
572
368
  ...
573
369
 
574
370
  @typing.overload
575
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
371
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
576
372
  """
577
- Specifies that the step will success under all circumstances.
373
+ Creates a human-readable report, a Metaflow Card, after this step completes.
578
374
 
579
- The decorator will create an optional artifact, specified by `var`, which
580
- contains the exception raised. You can use it to detect the presence
581
- of errors, indicating that all happy-path artifacts produced by the step
582
- are missing.
375
+ Note that you may add multiple `@card` decorators in a step with different parameters.
583
376
 
584
377
 
585
378
  Parameters
586
379
  ----------
587
- var : str, optional, default None
588
- Name of the artifact in which to store the caught exception.
589
- If not specified, the exception is not stored.
590
- print_exception : bool, default True
591
- Determines whether or not the exception is printed to
592
- stdout when caught.
380
+ type : str, default 'default'
381
+ Card type.
382
+ id : str, optional, default None
383
+ If multiple cards are present, use this id to identify this card.
384
+ options : Dict[str, Any], default {}
385
+ Options passed to the card. The contents depend on the card type.
386
+ timeout : int, default 45
387
+ Interrupt reporting if it takes more than this many seconds.
593
388
  """
594
389
  ...
595
390
 
596
391
  @typing.overload
597
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
392
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
598
393
  ...
599
394
 
600
395
  @typing.overload
601
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
396
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
602
397
  ...
603
398
 
604
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
399
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
605
400
  """
606
- Specifies that the step will success under all circumstances.
401
+ Creates a human-readable report, a Metaflow Card, after this step completes.
607
402
 
608
- The decorator will create an optional artifact, specified by `var`, which
609
- contains the exception raised. You can use it to detect the presence
610
- of errors, indicating that all happy-path artifacts produced by the step
611
- are missing.
612
-
613
-
614
- Parameters
615
- ----------
616
- var : str, optional, default None
617
- Name of the artifact in which to store the caught exception.
618
- If not specified, the exception is not stored.
619
- print_exception : bool, default True
620
- Determines whether or not the exception is printed to
621
- stdout when caught.
622
- """
623
- ...
624
-
625
- @typing.overload
626
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
627
- """
628
- Creates a human-readable report, a Metaflow Card, after this step completes.
629
-
630
- Note that you may add multiple `@card` decorators in a step with different parameters.
403
+ Note that you may add multiple `@card` decorators in a step with different parameters.
631
404
 
632
405
 
633
406
  Parameters
@@ -644,30 +417,309 @@ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typ
644
417
  ...
645
418
 
646
419
  @typing.overload
647
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
420
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
421
+ """
422
+ Decorator prototype for all step decorators. This function gets specialized
423
+ and imported for all decorators types by _import_plugin_decorators().
424
+ """
648
425
  ...
649
426
 
650
427
  @typing.overload
651
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
428
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
652
429
  ...
653
430
 
654
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
431
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
655
432
  """
656
- Creates a human-readable report, a Metaflow Card, after this step completes.
433
+ Decorator prototype for all step decorators. This function gets specialized
434
+ and imported for all decorators types by _import_plugin_decorators().
435
+ """
436
+ ...
437
+
438
+ @typing.overload
439
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
440
+ """
441
+ Specifies that the step will success under all circumstances.
657
442
 
658
- Note that you may add multiple `@card` decorators in a step with different parameters.
443
+ The decorator will create an optional artifact, specified by `var`, which
444
+ contains the exception raised. You can use it to detect the presence
445
+ of errors, indicating that all happy-path artifacts produced by the step
446
+ are missing.
659
447
 
660
448
 
661
449
  Parameters
662
450
  ----------
663
- type : str, default 'default'
664
- Card type.
665
- id : str, optional, default None
666
- If multiple cards are present, use this id to identify this card.
667
- options : Dict[str, Any], default {}
668
- Options passed to the card. The contents depend on the card type.
669
- timeout : int, default 45
670
- Interrupt reporting if it takes more than this many seconds.
451
+ var : str, optional, default None
452
+ Name of the artifact in which to store the caught exception.
453
+ If not specified, the exception is not stored.
454
+ print_exception : bool, default True
455
+ Determines whether or not the exception is printed to
456
+ stdout when caught.
457
+ """
458
+ ...
459
+
460
+ @typing.overload
461
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
462
+ ...
463
+
464
+ @typing.overload
465
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
466
+ ...
467
+
468
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
469
+ """
470
+ Specifies that the step will success under all circumstances.
471
+
472
+ The decorator will create an optional artifact, specified by `var`, which
473
+ contains the exception raised. You can use it to detect the presence
474
+ of errors, indicating that all happy-path artifacts produced by the step
475
+ are missing.
476
+
477
+
478
+ Parameters
479
+ ----------
480
+ var : str, optional, default None
481
+ Name of the artifact in which to store the caught exception.
482
+ If not specified, the exception is not stored.
483
+ print_exception : bool, default True
484
+ Determines whether or not the exception is printed to
485
+ stdout when caught.
486
+ """
487
+ ...
488
+
489
+ @typing.overload
490
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
491
+ """
492
+ Specifies the number of times the task corresponding
493
+ to a step needs to be retried.
494
+
495
+ This decorator is useful for handling transient errors, such as networking issues.
496
+ If your task contains operations that can't be retried safely, e.g. database updates,
497
+ it is advisable to annotate it with `@retry(times=0)`.
498
+
499
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
500
+ decorator will execute a no-op task after all retries have been exhausted,
501
+ ensuring that the flow execution can continue.
502
+
503
+
504
+ Parameters
505
+ ----------
506
+ times : int, default 3
507
+ Number of times to retry this task.
508
+ minutes_between_retries : int, default 2
509
+ Number of minutes between retries.
510
+ """
511
+ ...
512
+
513
+ @typing.overload
514
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
515
+ ...
516
+
517
+ @typing.overload
518
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
519
+ ...
520
+
521
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
522
+ """
523
+ Specifies the number of times the task corresponding
524
+ to a step needs to be retried.
525
+
526
+ This decorator is useful for handling transient errors, such as networking issues.
527
+ If your task contains operations that can't be retried safely, e.g. database updates,
528
+ it is advisable to annotate it with `@retry(times=0)`.
529
+
530
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
531
+ decorator will execute a no-op task after all retries have been exhausted,
532
+ ensuring that the flow execution can continue.
533
+
534
+
535
+ Parameters
536
+ ----------
537
+ times : int, default 3
538
+ Number of times to retry this task.
539
+ minutes_between_retries : int, default 2
540
+ Number of minutes between retries.
541
+ """
542
+ ...
543
+
544
+ @typing.overload
545
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
546
+ """
547
+ Specifies environment variables to be set prior to the execution of a step.
548
+
549
+
550
+ Parameters
551
+ ----------
552
+ vars : Dict[str, str], default {}
553
+ Dictionary of environment variables to set.
554
+ """
555
+ ...
556
+
557
+ @typing.overload
558
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
559
+ ...
560
+
561
+ @typing.overload
562
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
563
+ ...
564
+
565
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
566
+ """
567
+ Specifies environment variables to be set prior to the execution of a step.
568
+
569
+
570
+ Parameters
571
+ ----------
572
+ vars : Dict[str, str], default {}
573
+ Dictionary of environment variables to set.
574
+ """
575
+ ...
576
+
577
+ @typing.overload
578
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
579
+ """
580
+ Specifies the PyPI packages for the step.
581
+
582
+ Information in this decorator will augment any
583
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
584
+ you can use `@pypi_base` to set packages required by all
585
+ steps and use `@pypi` to specify step-specific overrides.
586
+
587
+
588
+ Parameters
589
+ ----------
590
+ packages : Dict[str, str], default: {}
591
+ Packages to use for this step. The key is the name of the package
592
+ and the value is the version to use.
593
+ python : str, optional, default: None
594
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
595
+ that the version used will correspond to the version of the Python interpreter used to start the run.
596
+ """
597
+ ...
598
+
599
+ @typing.overload
600
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
601
+ ...
602
+
603
+ @typing.overload
604
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
605
+ ...
606
+
607
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
608
+ """
609
+ Specifies the PyPI packages for the step.
610
+
611
+ Information in this decorator will augment any
612
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
613
+ you can use `@pypi_base` to set packages required by all
614
+ steps and use `@pypi` to specify step-specific overrides.
615
+
616
+
617
+ Parameters
618
+ ----------
619
+ packages : Dict[str, str], default: {}
620
+ Packages to use for this step. The key is the name of the package
621
+ and the value is the version to use.
622
+ python : str, optional, default: None
623
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
624
+ that the version used will correspond to the version of the Python interpreter used to start the run.
625
+ """
626
+ ...
627
+
628
+ @typing.overload
629
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
630
+ """
631
+ Specifies secrets to be retrieved and injected as environment variables prior to
632
+ the execution of a step.
633
+
634
+
635
+ Parameters
636
+ ----------
637
+ sources : List[Union[str, Dict[str, Any]]], default: []
638
+ List of secret specs, defining how the secrets are to be retrieved
639
+ role : str, optional, default: None
640
+ Role to use for fetching secrets
641
+ """
642
+ ...
643
+
644
+ @typing.overload
645
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
646
+ ...
647
+
648
+ @typing.overload
649
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
650
+ ...
651
+
652
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
653
+ """
654
+ Specifies secrets to be retrieved and injected as environment variables prior to
655
+ the execution of a step.
656
+
657
+
658
+ Parameters
659
+ ----------
660
+ sources : List[Union[str, Dict[str, Any]]], default: []
661
+ List of secret specs, defining how the secrets are to be retrieved
662
+ role : str, optional, default: None
663
+ Role to use for fetching secrets
664
+ """
665
+ ...
666
+
667
+ @typing.overload
668
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
669
+ """
670
+ Specifies the Conda environment for the step.
671
+
672
+ Information in this decorator will augment any
673
+ attributes set in the `@conda_base` flow-level decorator. Hence,
674
+ you can use `@conda_base` to set packages required by all
675
+ steps and use `@conda` to specify step-specific overrides.
676
+
677
+
678
+ Parameters
679
+ ----------
680
+ packages : Dict[str, str], default {}
681
+ Packages to use for this step. The key is the name of the package
682
+ and the value is the version to use.
683
+ libraries : Dict[str, str], default {}
684
+ Supported for backward compatibility. When used with packages, packages will take precedence.
685
+ python : str, optional, default None
686
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
687
+ that the version used will correspond to the version of the Python interpreter used to start the run.
688
+ disabled : bool, default False
689
+ If set to True, disables @conda.
690
+ """
691
+ ...
692
+
693
+ @typing.overload
694
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
695
+ ...
696
+
697
+ @typing.overload
698
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
699
+ ...
700
+
701
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
702
+ """
703
+ Specifies the Conda environment for the step.
704
+
705
+ Information in this decorator will augment any
706
+ attributes set in the `@conda_base` flow-level decorator. Hence,
707
+ you can use `@conda_base` to set packages required by all
708
+ steps and use `@conda` to specify step-specific overrides.
709
+
710
+
711
+ Parameters
712
+ ----------
713
+ packages : Dict[str, str], default {}
714
+ Packages to use for this step. The key is the name of the package
715
+ and the value is the version to use.
716
+ libraries : Dict[str, str], default {}
717
+ Supported for backward compatibility. When used with packages, packages will take precedence.
718
+ python : str, optional, default None
719
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
720
+ that the version used will correspond to the version of the Python interpreter used to start the run.
721
+ disabled : bool, default False
722
+ If set to True, disables @conda.
671
723
  """
672
724
  ...
673
725
 
@@ -746,68 +798,17 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
746
798
  If not specified, any accessible compute pool within the perimeter is used.
747
799
  hostname_resolution_timeout: int, default 10 * 60
748
800
  Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
749
- Only applicable when @parallel is used.
750
- qos: str, default: Burstable
751
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
752
-
753
- security_context: Dict[str, Any], optional, default None
754
- Container security context. Applies to the task container. Allows the following keys:
755
- - privileged: bool, optional, default None
756
- - allow_privilege_escalation: bool, optional, default None
757
- - run_as_user: int, optional, default None
758
- - run_as_group: int, optional, default None
759
- - run_as_non_root: bool, optional, default None
760
- """
761
- ...
762
-
763
- @typing.overload
764
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
765
- """
766
- Specifies the PyPI packages for the step.
767
-
768
- Information in this decorator will augment any
769
- attributes set in the `@pyi_base` flow-level decorator. Hence,
770
- you can use `@pypi_base` to set packages required by all
771
- steps and use `@pypi` to specify step-specific overrides.
772
-
773
-
774
- Parameters
775
- ----------
776
- packages : Dict[str, str], default: {}
777
- Packages to use for this step. The key is the name of the package
778
- and the value is the version to use.
779
- python : str, optional, default: None
780
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
781
- that the version used will correspond to the version of the Python interpreter used to start the run.
782
- """
783
- ...
784
-
785
- @typing.overload
786
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
787
- ...
788
-
789
- @typing.overload
790
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
791
- ...
792
-
793
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
794
- """
795
- Specifies the PyPI packages for the step.
796
-
797
- Information in this decorator will augment any
798
- attributes set in the `@pyi_base` flow-level decorator. Hence,
799
- you can use `@pypi_base` to set packages required by all
800
- steps and use `@pypi` to specify step-specific overrides.
801
-
801
+ Only applicable when @parallel is used.
802
+ qos: str, default: Burstable
803
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
802
804
 
803
- Parameters
804
- ----------
805
- packages : Dict[str, str], default: {}
806
- Packages to use for this step. The key is the name of the package
807
- and the value is the version to use.
808
- python : str, optional, default: None
809
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
810
- that the version used will correspond to the version of the Python interpreter used to start the run.
805
+ security_context: Dict[str, Any], optional, default None
806
+ Container security context. Applies to the task container. Allows the following keys:
807
+ - privileged: bool, optional, default None
808
+ - allow_privilege_escalation: bool, optional, default None
809
+ - run_as_user: int, optional, default None
810
+ - run_as_group: int, optional, default None
811
+ - run_as_non_root: bool, optional, default None
811
812
  """
812
813
  ...
813
814
 
@@ -890,6 +891,148 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
890
891
  """
891
892
  ...
892
893
 
894
+ @typing.overload
895
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
896
+ """
897
+ Specifies the PyPI packages for all steps of the flow.
898
+
899
+ Use `@pypi_base` to set common packages required by all
900
+ steps and use `@pypi` to specify step-specific overrides.
901
+
902
+ Parameters
903
+ ----------
904
+ packages : Dict[str, str], default: {}
905
+ Packages to use for this flow. The key is the name of the package
906
+ and the value is the version to use.
907
+ python : str, optional, default: None
908
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
909
+ that the version used will correspond to the version of the Python interpreter used to start the run.
910
+ """
911
+ ...
912
+
913
+ @typing.overload
914
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
915
+ ...
916
+
917
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
918
+ """
919
+ Specifies the PyPI packages for all steps of the flow.
920
+
921
+ Use `@pypi_base` to set common packages required by all
922
+ steps and use `@pypi` to specify step-specific overrides.
923
+
924
+ Parameters
925
+ ----------
926
+ packages : Dict[str, str], default: {}
927
+ Packages to use for this flow. The key is the name of the package
928
+ and the value is the version to use.
929
+ python : str, optional, default: None
930
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
931
+ that the version used will correspond to the version of the Python interpreter used to start the run.
932
+ """
933
+ ...
934
+
935
+ @typing.overload
936
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
937
+ """
938
+ Specifies the flow(s) that this flow depends on.
939
+
940
+ ```
941
+ @trigger_on_finish(flow='FooFlow')
942
+ ```
943
+ or
944
+ ```
945
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
946
+ ```
947
+ This decorator respects the @project decorator and triggers the flow
948
+ when upstream runs within the same namespace complete successfully
949
+
950
+ Additionally, you can specify project aware upstream flow dependencies
951
+ by specifying the fully qualified project_flow_name.
952
+ ```
953
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
954
+ ```
955
+ or
956
+ ```
957
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
958
+ ```
959
+
960
+ You can also specify just the project or project branch (other values will be
961
+ inferred from the current project or project branch):
962
+ ```
963
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
964
+ ```
965
+
966
+ Note that `branch` is typically one of:
967
+ - `prod`
968
+ - `user.bob`
969
+ - `test.my_experiment`
970
+ - `prod.staging`
971
+
972
+
973
+ Parameters
974
+ ----------
975
+ flow : Union[str, Dict[str, str]], optional, default None
976
+ Upstream flow dependency for this flow.
977
+ flows : List[Union[str, Dict[str, str]]], default []
978
+ Upstream flow dependencies for this flow.
979
+ options : Dict[str, Any], default {}
980
+ Backend-specific configuration for tuning eventing behavior.
981
+ """
982
+ ...
983
+
984
+ @typing.overload
985
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
986
+ ...
987
+
988
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
989
+ """
990
+ Specifies the flow(s) that this flow depends on.
991
+
992
+ ```
993
+ @trigger_on_finish(flow='FooFlow')
994
+ ```
995
+ or
996
+ ```
997
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
998
+ ```
999
+ This decorator respects the @project decorator and triggers the flow
1000
+ when upstream runs within the same namespace complete successfully
1001
+
1002
+ Additionally, you can specify project aware upstream flow dependencies
1003
+ by specifying the fully qualified project_flow_name.
1004
+ ```
1005
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1006
+ ```
1007
+ or
1008
+ ```
1009
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1010
+ ```
1011
+
1012
+ You can also specify just the project or project branch (other values will be
1013
+ inferred from the current project or project branch):
1014
+ ```
1015
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1016
+ ```
1017
+
1018
+ Note that `branch` is typically one of:
1019
+ - `prod`
1020
+ - `user.bob`
1021
+ - `test.my_experiment`
1022
+ - `prod.staging`
1023
+
1024
+
1025
+ Parameters
1026
+ ----------
1027
+ flow : Union[str, Dict[str, str]], optional, default None
1028
+ Upstream flow dependency for this flow.
1029
+ flows : List[Union[str, Dict[str, str]]], default []
1030
+ Upstream flow dependencies for this flow.
1031
+ options : Dict[str, Any], default {}
1032
+ Backend-specific configuration for tuning eventing behavior.
1033
+ """
1034
+ ...
1035
+
893
1036
  @typing.overload
894
1037
  def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
895
1038
  """
@@ -941,104 +1084,132 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
941
1084
  """
942
1085
  ...
943
1086
 
944
- @typing.overload
945
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1087
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
946
1088
  """
947
- Specifies the flow(s) that this flow depends on.
1089
+ Specifies what flows belong to the same project.
948
1090
 
949
- ```
950
- @trigger_on_finish(flow='FooFlow')
951
- ```
952
- or
953
- ```
954
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
955
- ```
956
- This decorator respects the @project decorator and triggers the flow
957
- when upstream runs within the same namespace complete successfully
1091
+ A project-specific namespace is created for all flows that
1092
+ use the same `@project(name)`.
958
1093
 
959
- Additionally, you can specify project aware upstream flow dependencies
960
- by specifying the fully qualified project_flow_name.
961
- ```
962
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
963
- ```
964
- or
965
- ```
966
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
967
- ```
968
1094
 
969
- You can also specify just the project or project branch (other values will be
970
- inferred from the current project or project branch):
971
- ```
972
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
973
- ```
1095
+ Parameters
1096
+ ----------
1097
+ name : str
1098
+ Project name. Make sure that the name is unique amongst all
1099
+ projects that use the same production scheduler. The name may
1100
+ contain only lowercase alphanumeric characters and underscores.
1101
+
1102
+ branch : Optional[str], default None
1103
+ The branch to use. If not specified, the branch is set to
1104
+ `user.<username>` unless `production` is set to `True`. This can
1105
+ also be set on the command line using `--branch` as a top-level option.
1106
+ It is an error to specify `branch` in the decorator and on the command line.
1107
+
1108
+ production : bool, default False
1109
+ Whether or not the branch is the production branch. This can also be set on the
1110
+ command line using `--production` as a top-level option. It is an error to specify
1111
+ `production` in the decorator and on the command line.
1112
+ The project branch name will be:
1113
+ - if `branch` is specified:
1114
+ - if `production` is True: `prod.<branch>`
1115
+ - if `production` is False: `test.<branch>`
1116
+ - if `branch` is not specified:
1117
+ - if `production` is True: `prod`
1118
+ - if `production` is False: `user.<username>`
1119
+ """
1120
+ ...
1121
+
1122
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1123
+ """
1124
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1125
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1126
+
1127
+
1128
+ Parameters
1129
+ ----------
1130
+ timeout : int
1131
+ Time, in seconds before the task times out and fails. (Default: 3600)
1132
+ poke_interval : int
1133
+ Time in seconds that the job should wait in between each try. (Default: 60)
1134
+ mode : str
1135
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1136
+ exponential_backoff : bool
1137
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1138
+ pool : str
1139
+ the slot pool this task should run in,
1140
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1141
+ soft_fail : bool
1142
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1143
+ name : str
1144
+ Name of the sensor on Airflow
1145
+ description : str
1146
+ Description of sensor in the Airflow UI
1147
+ external_dag_id : str
1148
+ The dag_id that contains the task you want to wait for.
1149
+ external_task_ids : List[str]
1150
+ The list of task_ids that you want to wait for.
1151
+ If None (default value) the sensor waits for the DAG. (Default: None)
1152
+ allowed_states : List[str]
1153
+ Iterable of allowed states, (Default: ['success'])
1154
+ failed_states : List[str]
1155
+ Iterable of failed or dis-allowed states. (Default: None)
1156
+ execution_delta : datetime.timedelta
1157
+ time difference with the previous execution to look at,
1158
+ the default is the same logical date as the current task or DAG. (Default: None)
1159
+ check_existence: bool
1160
+ Set to True to check if the external task exists or check if
1161
+ the DAG to wait for exists. (Default: True)
1162
+ """
1163
+ ...
1164
+
1165
+ @typing.overload
1166
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1167
+ """
1168
+ Specifies the Conda environment for all steps of the flow.
974
1169
 
975
- Note that `branch` is typically one of:
976
- - `prod`
977
- - `user.bob`
978
- - `test.my_experiment`
979
- - `prod.staging`
1170
+ Use `@conda_base` to set common libraries required by all
1171
+ steps and use `@conda` to specify step-specific additions.
980
1172
 
981
1173
 
982
1174
  Parameters
983
1175
  ----------
984
- flow : Union[str, Dict[str, str]], optional, default None
985
- Upstream flow dependency for this flow.
986
- flows : List[Union[str, Dict[str, str]]], default []
987
- Upstream flow dependencies for this flow.
988
- options : Dict[str, Any], default {}
989
- Backend-specific configuration for tuning eventing behavior.
1176
+ packages : Dict[str, str], default {}
1177
+ Packages to use for this flow. The key is the name of the package
1178
+ and the value is the version to use.
1179
+ libraries : Dict[str, str], default {}
1180
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1181
+ python : str, optional, default None
1182
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1183
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1184
+ disabled : bool, default False
1185
+ If set to True, disables Conda.
990
1186
  """
991
1187
  ...
992
1188
 
993
1189
  @typing.overload
994
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1190
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
995
1191
  ...
996
1192
 
997
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1193
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
998
1194
  """
999
- Specifies the flow(s) that this flow depends on.
1000
-
1001
- ```
1002
- @trigger_on_finish(flow='FooFlow')
1003
- ```
1004
- or
1005
- ```
1006
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1007
- ```
1008
- This decorator respects the @project decorator and triggers the flow
1009
- when upstream runs within the same namespace complete successfully
1010
-
1011
- Additionally, you can specify project aware upstream flow dependencies
1012
- by specifying the fully qualified project_flow_name.
1013
- ```
1014
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1015
- ```
1016
- or
1017
- ```
1018
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1019
- ```
1020
-
1021
- You can also specify just the project or project branch (other values will be
1022
- inferred from the current project or project branch):
1023
- ```
1024
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1025
- ```
1195
+ Specifies the Conda environment for all steps of the flow.
1026
1196
 
1027
- Note that `branch` is typically one of:
1028
- - `prod`
1029
- - `user.bob`
1030
- - `test.my_experiment`
1031
- - `prod.staging`
1197
+ Use `@conda_base` to set common libraries required by all
1198
+ steps and use `@conda` to specify step-specific additions.
1032
1199
 
1033
1200
 
1034
1201
  Parameters
1035
1202
  ----------
1036
- flow : Union[str, Dict[str, str]], optional, default None
1037
- Upstream flow dependency for this flow.
1038
- flows : List[Union[str, Dict[str, str]]], default []
1039
- Upstream flow dependencies for this flow.
1040
- options : Dict[str, Any], default {}
1041
- Backend-specific configuration for tuning eventing behavior.
1203
+ packages : Dict[str, str], default {}
1204
+ Packages to use for this flow. The key is the name of the package
1205
+ and the value is the version to use.
1206
+ libraries : Dict[str, str], default {}
1207
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1208
+ python : str, optional, default None
1209
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1210
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1211
+ disabled : bool, default False
1212
+ If set to True, disables Conda.
1042
1213
  """
1043
1214
  ...
1044
1215
 
@@ -1178,173 +1349,3 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1178
1349
  """
1179
1350
  ...
1180
1351
 
1181
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1182
- """
1183
- Specifies what flows belong to the same project.
1184
-
1185
- A project-specific namespace is created for all flows that
1186
- use the same `@project(name)`.
1187
-
1188
-
1189
- Parameters
1190
- ----------
1191
- name : str
1192
- Project name. Make sure that the name is unique amongst all
1193
- projects that use the same production scheduler. The name may
1194
- contain only lowercase alphanumeric characters and underscores.
1195
-
1196
- branch : Optional[str], default None
1197
- The branch to use. If not specified, the branch is set to
1198
- `user.<username>` unless `production` is set to `True`. This can
1199
- also be set on the command line using `--branch` as a top-level option.
1200
- It is an error to specify `branch` in the decorator and on the command line.
1201
-
1202
- production : bool, default False
1203
- Whether or not the branch is the production branch. This can also be set on the
1204
- command line using `--production` as a top-level option. It is an error to specify
1205
- `production` in the decorator and on the command line.
1206
- The project branch name will be:
1207
- - if `branch` is specified:
1208
- - if `production` is True: `prod.<branch>`
1209
- - if `production` is False: `test.<branch>`
1210
- - if `branch` is not specified:
1211
- - if `production` is True: `prod`
1212
- - if `production` is False: `user.<username>`
1213
- """
1214
- ...
1215
-
1216
- @typing.overload
1217
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1218
- """
1219
- Specifies the PyPI packages for all steps of the flow.
1220
-
1221
- Use `@pypi_base` to set common packages required by all
1222
- steps and use `@pypi` to specify step-specific overrides.
1223
-
1224
- Parameters
1225
- ----------
1226
- packages : Dict[str, str], default: {}
1227
- Packages to use for this flow. The key is the name of the package
1228
- and the value is the version to use.
1229
- python : str, optional, default: None
1230
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1231
- that the version used will correspond to the version of the Python interpreter used to start the run.
1232
- """
1233
- ...
1234
-
1235
- @typing.overload
1236
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1237
- ...
1238
-
1239
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1240
- """
1241
- Specifies the PyPI packages for all steps of the flow.
1242
-
1243
- Use `@pypi_base` to set common packages required by all
1244
- steps and use `@pypi` to specify step-specific overrides.
1245
-
1246
- Parameters
1247
- ----------
1248
- packages : Dict[str, str], default: {}
1249
- Packages to use for this flow. The key is the name of the package
1250
- and the value is the version to use.
1251
- python : str, optional, default: None
1252
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1253
- that the version used will correspond to the version of the Python interpreter used to start the run.
1254
- """
1255
- ...
1256
-
1257
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1258
- """
1259
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1260
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1261
-
1262
-
1263
- Parameters
1264
- ----------
1265
- timeout : int
1266
- Time, in seconds before the task times out and fails. (Default: 3600)
1267
- poke_interval : int
1268
- Time in seconds that the job should wait in between each try. (Default: 60)
1269
- mode : str
1270
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1271
- exponential_backoff : bool
1272
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1273
- pool : str
1274
- the slot pool this task should run in,
1275
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1276
- soft_fail : bool
1277
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1278
- name : str
1279
- Name of the sensor on Airflow
1280
- description : str
1281
- Description of sensor in the Airflow UI
1282
- external_dag_id : str
1283
- The dag_id that contains the task you want to wait for.
1284
- external_task_ids : List[str]
1285
- The list of task_ids that you want to wait for.
1286
- If None (default value) the sensor waits for the DAG. (Default: None)
1287
- allowed_states : List[str]
1288
- Iterable of allowed states, (Default: ['success'])
1289
- failed_states : List[str]
1290
- Iterable of failed or dis-allowed states. (Default: None)
1291
- execution_delta : datetime.timedelta
1292
- time difference with the previous execution to look at,
1293
- the default is the same logical date as the current task or DAG. (Default: None)
1294
- check_existence: bool
1295
- Set to True to check if the external task exists or check if
1296
- the DAG to wait for exists. (Default: True)
1297
- """
1298
- ...
1299
-
1300
- @typing.overload
1301
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1302
- """
1303
- Specifies the Conda environment for all steps of the flow.
1304
-
1305
- Use `@conda_base` to set common libraries required by all
1306
- steps and use `@conda` to specify step-specific additions.
1307
-
1308
-
1309
- Parameters
1310
- ----------
1311
- packages : Dict[str, str], default {}
1312
- Packages to use for this flow. The key is the name of the package
1313
- and the value is the version to use.
1314
- libraries : Dict[str, str], default {}
1315
- Supported for backward compatibility. When used with packages, packages will take precedence.
1316
- python : str, optional, default None
1317
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1318
- that the version used will correspond to the version of the Python interpreter used to start the run.
1319
- disabled : bool, default False
1320
- If set to True, disables Conda.
1321
- """
1322
- ...
1323
-
1324
- @typing.overload
1325
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1326
- ...
1327
-
1328
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1329
- """
1330
- Specifies the Conda environment for all steps of the flow.
1331
-
1332
- Use `@conda_base` to set common libraries required by all
1333
- steps and use `@conda` to specify step-specific additions.
1334
-
1335
-
1336
- Parameters
1337
- ----------
1338
- packages : Dict[str, str], default {}
1339
- Packages to use for this flow. The key is the name of the package
1340
- and the value is the version to use.
1341
- libraries : Dict[str, str], default {}
1342
- Supported for backward compatibility. When used with packages, packages will take precedence.
1343
- python : str, optional, default None
1344
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1345
- that the version used will correspond to the version of the Python interpreter used to start the run.
1346
- disabled : bool, default False
1347
- If set to True, disables Conda.
1348
- """
1349
- ...
1350
-