metaflow-stubs 2.12.38__py2.py3-none-any.whl → 2.13__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (144) hide show
  1. metaflow-stubs/__init__.pyi +222 -221
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +3 -3
  11. metaflow-stubs/flowspec.pyi +6 -6
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +2 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +4 -2
  20. metaflow-stubs/metaflow_current.pyi +23 -23
  21. metaflow-stubs/multicore_utils.pyi +2 -2
  22. metaflow-stubs/parameters.pyi +4 -4
  23. metaflow-stubs/plugins/__init__.pyi +12 -12
  24. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  31. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  32. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  35. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  37. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  38. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +30 -2
  39. metaflow-stubs/plugins/aws/__init__.pyi +3 -3
  40. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  47. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +15 -13
  48. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  57. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  61. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  62. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  63. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  64. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  66. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_decorator.pyi +4 -2
  69. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  73. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  77. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  78. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  79. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  80. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  82. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  83. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  84. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  85. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  86. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  87. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  88. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  89. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  90. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  94. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  95. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  96. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  97. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  98. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  100. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  101. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  102. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  105. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  109. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  110. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  111. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  112. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  113. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  114. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  116. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  117. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  119. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  120. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  121. metaflow-stubs/pylint_wrapper.pyi +2 -2
  122. metaflow-stubs/runner/__init__.pyi +2 -2
  123. metaflow-stubs/runner/deployer.pyi +6 -6
  124. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  125. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  126. metaflow-stubs/runner/nbdeploy.pyi +4 -2
  127. metaflow-stubs/runner/nbrun.pyi +3 -3
  128. metaflow-stubs/runner/subprocess_manager.pyi +5 -2
  129. metaflow-stubs/runner/utils.pyi +8 -8
  130. metaflow-stubs/system/__init__.pyi +2 -2
  131. metaflow-stubs/system/system_logger.pyi +2 -2
  132. metaflow-stubs/system/system_monitor.pyi +2 -2
  133. metaflow-stubs/tagging_util.pyi +2 -2
  134. metaflow-stubs/tuple_util.pyi +2 -2
  135. metaflow-stubs/user_configs/__init__.pyi +2 -2
  136. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  137. metaflow-stubs/user_configs/config_options.pyi +8 -3
  138. metaflow-stubs/user_configs/config_parameters.pyi +8 -8
  139. metaflow-stubs/version.pyi +2 -2
  140. {metaflow_stubs-2.12.38.dist-info → metaflow_stubs-2.13.dist-info}/METADATA +2 -2
  141. metaflow_stubs-2.13.dist-info/RECORD +144 -0
  142. metaflow_stubs-2.12.38.dist-info/RECORD +0 -144
  143. {metaflow_stubs-2.12.38.dist-info → metaflow_stubs-2.13.dist-info}/WHEEL +0 -0
  144. {metaflow_stubs-2.12.38.dist-info → metaflow_stubs-2.13.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.38 #
4
- # Generated on 2024-12-08T03:54:58.005048 #
3
+ # MF version: 2.13 #
4
+ # Generated on 2024-12-20T07:38:30.145823 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import typing
12
11
  import datetime
12
+ import typing
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -31,6 +31,7 @@ from .parameters import Parameter as Parameter
31
31
  from .parameters import JSONTypeClass as JSONTypeClass
32
32
  from .parameters import JSONType as JSONType
33
33
  from .user_configs.config_parameters import Config as Config
34
+ from .user_configs.config_parameters import ConfigValue as ConfigValue
34
35
  from .user_configs.config_parameters import config_expr as config_expr
35
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
36
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
@@ -141,115 +142,94 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
141
142
  """
142
143
  ...
143
144
 
144
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
145
- """
146
- Specifies that this step should execute on Kubernetes.
147
- """
148
- ...
149
-
150
145
  @typing.overload
151
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
146
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
152
147
  """
153
- Specifies the PyPI packages for the step.
148
+ Specifies a timeout for your step.
154
149
 
155
- Information in this decorator will augment any
156
- attributes set in the `@pyi_base` flow-level decorator. Hence,
157
- you can use `@pypi_base` to set packages required by all
158
- steps and use `@pypi` to specify step-specific overrides.
150
+ This decorator is useful if this step may hang indefinitely.
151
+
152
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
153
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
154
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
155
+
156
+ Note that all the values specified in parameters are added together so if you specify
157
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
159
158
  """
160
159
  ...
161
160
 
162
161
  @typing.overload
163
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
162
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
164
163
  ...
165
164
 
166
165
  @typing.overload
167
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
166
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
168
167
  ...
169
168
 
170
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
169
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
171
170
  """
172
- Specifies the PyPI packages for the step.
171
+ Specifies a timeout for your step.
173
172
 
174
- Information in this decorator will augment any
175
- attributes set in the `@pyi_base` flow-level decorator. Hence,
176
- you can use `@pypi_base` to set packages required by all
177
- steps and use `@pypi` to specify step-specific overrides.
173
+ This decorator is useful if this step may hang indefinitely.
174
+
175
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
176
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
177
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
178
+
179
+ Note that all the values specified in parameters are added together so if you specify
180
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
178
181
  """
179
182
  ...
180
183
 
181
184
  @typing.overload
182
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
185
+ def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
183
186
  """
184
- Specifies the Conda environment for the step.
185
-
186
- Information in this decorator will augment any
187
- attributes set in the `@conda_base` flow-level decorator. Hence,
188
- you can use `@conda_base` to set packages required by all
189
- steps and use `@conda` to specify step-specific overrides.
187
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
190
188
  """
191
189
  ...
192
190
 
193
191
  @typing.overload
194
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
192
+ def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
195
193
  ...
196
194
 
197
195
  @typing.overload
198
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
196
+ def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
199
197
  ...
200
198
 
201
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
199
+ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
202
200
  """
203
- Specifies the Conda environment for the step.
204
-
205
- Information in this decorator will augment any
206
- attributes set in the `@conda_base` flow-level decorator. Hence,
207
- you can use `@conda_base` to set packages required by all
208
- steps and use `@conda` to specify step-specific overrides.
201
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
209
202
  """
210
203
  ...
211
204
 
212
205
  @typing.overload
213
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
206
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
214
207
  """
215
- Creates a human-readable report, a Metaflow Card, after this step completes.
208
+ Specifies the PyPI packages for the step.
216
209
 
217
- Note that you may add multiple `@card` decorators in a step with different parameters.
210
+ Information in this decorator will augment any
211
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
212
+ you can use `@pypi_base` to set packages required by all
213
+ steps and use `@pypi` to specify step-specific overrides.
218
214
  """
219
215
  ...
220
216
 
221
217
  @typing.overload
222
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
218
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
223
219
  ...
224
220
 
225
221
  @typing.overload
226
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
222
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
227
223
  ...
228
224
 
229
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
225
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
230
226
  """
231
- Creates a human-readable report, a Metaflow Card, after this step completes.
227
+ Specifies the PyPI packages for the step.
232
228
 
233
- Note that you may add multiple `@card` decorators in a step with different parameters.
234
- """
235
- ...
236
-
237
- @typing.overload
238
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
239
- """
240
- Decorator prototype for all step decorators. This function gets specialized
241
- and imported for all decorators types by _import_plugin_decorators().
242
- """
243
- ...
244
-
245
- @typing.overload
246
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
247
- ...
248
-
249
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
250
- """
251
- Decorator prototype for all step decorators. This function gets specialized
252
- and imported for all decorators types by _import_plugin_decorators().
229
+ Information in this decorator will augment any
230
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
231
+ you can use `@pypi_base` to set packages required by all
232
+ steps and use `@pypi` to specify step-specific overrides.
253
233
  """
254
234
  ...
255
235
 
@@ -274,29 +254,6 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
274
254
  """
275
255
  ...
276
256
 
277
- @typing.overload
278
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
279
- """
280
- Specifies secrets to be retrieved and injected as environment variables prior to
281
- the execution of a step.
282
- """
283
- ...
284
-
285
- @typing.overload
286
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
287
- ...
288
-
289
- @typing.overload
290
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
291
- ...
292
-
293
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
294
- """
295
- Specifies secrets to be retrieved and injected as environment variables prior to
296
- the execution of a step.
297
- """
298
- ...
299
-
300
257
  @typing.overload
301
258
  def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
302
259
  """
@@ -336,6 +293,25 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
336
293
  """
337
294
  ...
338
295
 
296
+ @typing.overload
297
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
298
+ """
299
+ Decorator prototype for all step decorators. This function gets specialized
300
+ and imported for all decorators types by _import_plugin_decorators().
301
+ """
302
+ ...
303
+
304
+ @typing.overload
305
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
306
+ ...
307
+
308
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
309
+ """
310
+ Decorator prototype for all step decorators. This function gets specialized
311
+ and imported for all decorators types by _import_plugin_decorators().
312
+ """
313
+ ...
314
+
339
315
  @typing.overload
340
316
  def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
341
317
  """
@@ -386,133 +362,191 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
386
362
  ...
387
363
 
388
364
  @typing.overload
389
- def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
365
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
390
366
  """
391
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
367
+ Specifies secrets to be retrieved and injected as environment variables prior to
368
+ the execution of a step.
392
369
  """
393
370
  ...
394
371
 
395
372
  @typing.overload
396
- def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
373
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
397
374
  ...
398
375
 
399
376
  @typing.overload
400
- def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
377
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
401
378
  ...
402
379
 
403
- def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
380
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
404
381
  """
405
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
382
+ Specifies secrets to be retrieved and injected as environment variables prior to
383
+ the execution of a step.
384
+ """
385
+ ...
386
+
387
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
388
+ """
389
+ Specifies that this step should execute on Kubernetes.
406
390
  """
407
391
  ...
408
392
 
409
393
  @typing.overload
410
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
394
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
411
395
  """
412
- Specifies that the step will success under all circumstances.
396
+ Specifies the Conda environment for the step.
413
397
 
414
- The decorator will create an optional artifact, specified by `var`, which
415
- contains the exception raised. You can use it to detect the presence
416
- of errors, indicating that all happy-path artifacts produced by the step
417
- are missing.
398
+ Information in this decorator will augment any
399
+ attributes set in the `@conda_base` flow-level decorator. Hence,
400
+ you can use `@conda_base` to set packages required by all
401
+ steps and use `@conda` to specify step-specific overrides.
418
402
  """
419
403
  ...
420
404
 
421
405
  @typing.overload
422
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
406
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
423
407
  ...
424
408
 
425
409
  @typing.overload
426
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
410
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
427
411
  ...
428
412
 
429
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
413
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
430
414
  """
431
- Specifies that the step will success under all circumstances.
415
+ Specifies the Conda environment for the step.
432
416
 
433
- The decorator will create an optional artifact, specified by `var`, which
434
- contains the exception raised. You can use it to detect the presence
435
- of errors, indicating that all happy-path artifacts produced by the step
436
- are missing.
417
+ Information in this decorator will augment any
418
+ attributes set in the `@conda_base` flow-level decorator. Hence,
419
+ you can use `@conda_base` to set packages required by all
420
+ steps and use `@conda` to specify step-specific overrides.
437
421
  """
438
422
  ...
439
423
 
440
424
  @typing.overload
441
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
425
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
442
426
  """
443
- Specifies a timeout for your step.
444
-
445
- This decorator is useful if this step may hang indefinitely.
446
-
447
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
448
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
449
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
427
+ Creates a human-readable report, a Metaflow Card, after this step completes.
450
428
 
451
- Note that all the values specified in parameters are added together so if you specify
452
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
429
+ Note that you may add multiple `@card` decorators in a step with different parameters.
453
430
  """
454
431
  ...
455
432
 
456
433
  @typing.overload
457
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
434
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
458
435
  ...
459
436
 
460
437
  @typing.overload
461
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
438
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
462
439
  ...
463
440
 
464
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
441
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
465
442
  """
466
- Specifies a timeout for your step.
467
-
468
- This decorator is useful if this step may hang indefinitely.
469
-
470
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
471
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
472
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
443
+ Creates a human-readable report, a Metaflow Card, after this step completes.
473
444
 
474
- Note that all the values specified in parameters are added together so if you specify
475
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
445
+ Note that you may add multiple `@card` decorators in a step with different parameters.
476
446
  """
477
447
  ...
478
448
 
479
449
  @typing.overload
480
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
450
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
481
451
  """
482
- Specifies the PyPI packages for all steps of the flow.
452
+ Specifies that the step will success under all circumstances.
483
453
 
484
- Use `@pypi_base` to set common packages required by all
485
- steps and use `@pypi` to specify step-specific overrides.
454
+ The decorator will create an optional artifact, specified by `var`, which
455
+ contains the exception raised. You can use it to detect the presence
456
+ of errors, indicating that all happy-path artifacts produced by the step
457
+ are missing.
486
458
  """
487
459
  ...
488
460
 
489
461
  @typing.overload
490
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
462
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
491
463
  ...
492
464
 
493
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
465
+ @typing.overload
466
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
467
+ ...
468
+
469
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
494
470
  """
495
- Specifies the PyPI packages for all steps of the flow.
471
+ Specifies that the step will success under all circumstances.
496
472
 
497
- Use `@pypi_base` to set common packages required by all
498
- steps and use `@pypi` to specify step-specific overrides.
473
+ The decorator will create an optional artifact, specified by `var`, which
474
+ contains the exception raised. You can use it to detect the presence
475
+ of errors, indicating that all happy-path artifacts produced by the step
476
+ are missing.
499
477
  """
500
478
  ...
501
479
 
502
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
480
+ @typing.overload
481
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
503
482
  """
504
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
505
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
483
+ Specifies the event(s) that this flow depends on.
484
+
485
+ ```
486
+ @trigger(event='foo')
487
+ ```
488
+ or
489
+ ```
490
+ @trigger(events=['foo', 'bar'])
491
+ ```
492
+
493
+ Additionally, you can specify the parameter mappings
494
+ to map event payload to Metaflow parameters for the flow.
495
+ ```
496
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
497
+ ```
498
+ or
499
+ ```
500
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
501
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
502
+ ```
503
+
504
+ 'parameters' can also be a list of strings and tuples like so:
505
+ ```
506
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
507
+ ```
508
+ This is equivalent to:
509
+ ```
510
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
511
+ ```
506
512
  """
507
513
  ...
508
514
 
509
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
515
+ @typing.overload
516
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
517
+ ...
518
+
519
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
510
520
  """
511
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
512
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
513
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
514
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
515
- starts only after all sensors finish.
521
+ Specifies the event(s) that this flow depends on.
522
+
523
+ ```
524
+ @trigger(event='foo')
525
+ ```
526
+ or
527
+ ```
528
+ @trigger(events=['foo', 'bar'])
529
+ ```
530
+
531
+ Additionally, you can specify the parameter mappings
532
+ to map event payload to Metaflow parameters for the flow.
533
+ ```
534
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
535
+ ```
536
+ or
537
+ ```
538
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
539
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
540
+ ```
541
+
542
+ 'parameters' can also be a list of strings and tuples like so:
543
+ ```
544
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
545
+ ```
546
+ This is equivalent to:
547
+ ```
548
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
549
+ ```
516
550
  """
517
551
  ...
518
552
 
@@ -597,6 +631,25 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
597
631
  """
598
632
  ...
599
633
 
634
+ @typing.overload
635
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
636
+ """
637
+ Specifies the times when the flow should be run when running on a
638
+ production scheduler.
639
+ """
640
+ ...
641
+
642
+ @typing.overload
643
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
644
+ ...
645
+
646
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
647
+ """
648
+ Specifies the times when the flow should be run when running on a
649
+ production scheduler.
650
+ """
651
+ ...
652
+
600
653
  @typing.overload
601
654
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
602
655
  """
@@ -621,94 +674,42 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
621
674
  ...
622
675
 
623
676
  @typing.overload
624
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
677
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
625
678
  """
626
- Specifies the times when the flow should be run when running on a
627
- production scheduler.
679
+ Specifies the PyPI packages for all steps of the flow.
680
+
681
+ Use `@pypi_base` to set common packages required by all
682
+ steps and use `@pypi` to specify step-specific overrides.
628
683
  """
629
684
  ...
630
685
 
631
686
  @typing.overload
632
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
687
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
633
688
  ...
634
689
 
635
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
690
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
636
691
  """
637
- Specifies the times when the flow should be run when running on a
638
- production scheduler.
692
+ Specifies the PyPI packages for all steps of the flow.
693
+
694
+ Use `@pypi_base` to set common packages required by all
695
+ steps and use `@pypi` to specify step-specific overrides.
639
696
  """
640
697
  ...
641
698
 
642
- @typing.overload
643
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
699
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
644
700
  """
645
- Specifies the event(s) that this flow depends on.
646
-
647
- ```
648
- @trigger(event='foo')
649
- ```
650
- or
651
- ```
652
- @trigger(events=['foo', 'bar'])
653
- ```
654
-
655
- Additionally, you can specify the parameter mappings
656
- to map event payload to Metaflow parameters for the flow.
657
- ```
658
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
659
- ```
660
- or
661
- ```
662
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
663
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
664
- ```
665
-
666
- 'parameters' can also be a list of strings and tuples like so:
667
- ```
668
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
669
- ```
670
- This is equivalent to:
671
- ```
672
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
673
- ```
701
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
702
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
703
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
704
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
705
+ starts only after all sensors finish.
674
706
  """
675
707
  ...
676
708
 
677
- @typing.overload
678
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
679
- ...
680
-
681
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
709
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
682
710
  """
683
- Specifies the event(s) that this flow depends on.
684
-
685
- ```
686
- @trigger(event='foo')
687
- ```
688
- or
689
- ```
690
- @trigger(events=['foo', 'bar'])
691
- ```
692
-
693
- Additionally, you can specify the parameter mappings
694
- to map event payload to Metaflow parameters for the flow.
695
- ```
696
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
697
- ```
698
- or
699
- ```
700
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
701
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
702
- ```
703
-
704
- 'parameters' can also be a list of strings and tuples like so:
705
- ```
706
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
707
- ```
708
- This is equivalent to:
709
- ```
710
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
711
- ```
711
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
712
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
712
713
  """
713
714
  ...
714
715