metaflow-stubs 2.16.7__py2.py3-none-any.whl → 2.16.8__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of metaflow-stubs might be problematic. Click here for more details.

Files changed (166) hide show
  1. metaflow-stubs/__init__.pyi +639 -639
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +5 -5
  14. metaflow-stubs/meta_files.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +18 -18
  21. metaflow-stubs/metaflow_git.pyi +2 -2
  22. metaflow-stubs/multicore_utils.pyi +2 -2
  23. metaflow-stubs/packaging_sys/__init__.pyi +6 -6
  24. metaflow-stubs/packaging_sys/backend.pyi +4 -4
  25. metaflow-stubs/packaging_sys/distribution_support.pyi +5 -5
  26. metaflow-stubs/packaging_sys/tar_backend.pyi +6 -6
  27. metaflow-stubs/packaging_sys/utils.pyi +2 -2
  28. metaflow-stubs/packaging_sys/v1.pyi +4 -4
  29. metaflow-stubs/parameters.pyi +5 -5
  30. metaflow-stubs/plugins/__init__.pyi +12 -12
  31. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  33. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  34. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  35. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  37. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  38. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  39. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  40. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  41. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  42. metaflow-stubs/plugins/argo/argo_workflows.pyi +33 -4
  43. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  44. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +5 -5
  45. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  46. metaflow-stubs/plugins/argo/exit_hooks.pyi +3 -3
  47. metaflow-stubs/plugins/aws/__init__.pyi +3 -3
  48. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  50. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  51. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  52. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  53. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  54. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  55. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +5 -5
  56. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  57. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  58. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  59. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  60. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  61. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  62. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  63. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  64. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +3 -3
  65. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  66. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  67. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  68. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +5 -5
  69. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  70. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  71. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  72. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  77. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  78. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  79. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  81. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  82. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  83. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  84. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  85. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  86. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  87. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  88. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  89. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  92. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  93. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  94. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  95. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  96. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  97. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  98. metaflow-stubs/plugins/exit_hook/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/exit_hook/exit_hook_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  103. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +5 -5
  104. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  105. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  106. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  107. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  108. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  109. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  110. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  111. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  112. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  114. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +3 -2
  115. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  116. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  117. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  119. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  120. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  121. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  122. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  123. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  124. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  125. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  126. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  127. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  128. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +4 -4
  129. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  130. metaflow-stubs/plugins/secrets/secrets_func.pyi +2 -2
  131. metaflow-stubs/plugins/secrets/secrets_spec.pyi +2 -2
  132. metaflow-stubs/plugins/secrets/utils.pyi +2 -2
  133. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  134. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  135. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  136. metaflow-stubs/plugins/uv/__init__.pyi +2 -2
  137. metaflow-stubs/plugins/uv/uv_environment.pyi +3 -3
  138. metaflow-stubs/pylint_wrapper.pyi +2 -2
  139. metaflow-stubs/runner/__init__.pyi +2 -2
  140. metaflow-stubs/runner/deployer.pyi +6 -6
  141. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  142. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  143. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  144. metaflow-stubs/runner/nbrun.pyi +2 -2
  145. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  146. metaflow-stubs/runner/utils.pyi +4 -4
  147. metaflow-stubs/system/__init__.pyi +2 -2
  148. metaflow-stubs/system/system_logger.pyi +3 -3
  149. metaflow-stubs/system/system_monitor.pyi +2 -2
  150. metaflow-stubs/tagging_util.pyi +2 -2
  151. metaflow-stubs/tuple_util.pyi +2 -2
  152. metaflow-stubs/user_configs/__init__.pyi +2 -2
  153. metaflow-stubs/user_configs/config_options.pyi +4 -4
  154. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  155. metaflow-stubs/user_decorators/__init__.pyi +2 -2
  156. metaflow-stubs/user_decorators/common.pyi +2 -2
  157. metaflow-stubs/user_decorators/mutable_flow.pyi +6 -6
  158. metaflow-stubs/user_decorators/mutable_step.pyi +5 -5
  159. metaflow-stubs/user_decorators/user_flow_decorator.pyi +5 -5
  160. metaflow-stubs/user_decorators/user_step_decorator.pyi +6 -6
  161. metaflow-stubs/version.pyi +2 -2
  162. {metaflow_stubs-2.16.7.dist-info → metaflow_stubs-2.16.8.dist-info}/METADATA +2 -2
  163. metaflow_stubs-2.16.8.dist-info/RECORD +166 -0
  164. metaflow_stubs-2.16.7.dist-info/RECORD +0 -166
  165. {metaflow_stubs-2.16.7.dist-info → metaflow_stubs-2.16.8.dist-info}/WHEEL +0 -0
  166. {metaflow_stubs-2.16.7.dist-info → metaflow_stubs-2.16.8.dist-info}/top_level.txt +0 -0
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.16.7 #
4
- # Generated on 2025-07-29T01:50:18.781263 #
3
+ # MF version: 2.16.8 #
4
+ # Generated on 2025-07-29T19:28:34.106017 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import datetime
12
11
  import typing
12
+ import datetime
13
13
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
14
14
  StepFlag = typing.NewType("StepFlag", bool)
15
15
 
@@ -47,9 +47,9 @@ from . import plugins as plugins
47
47
  from .plugins.datatools.s3.s3 import S3 as S3
48
48
  from . import includefile as includefile
49
49
  from .includefile import IncludeFile as IncludeFile
50
- from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
51
- from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
52
50
  from .plugins.pypi.parsers import requirements_txt_parser as requirements_txt_parser
51
+ from .plugins.pypi.parsers import conda_environment_yml_parser as conda_environment_yml_parser
52
+ from .plugins.pypi.parsers import pyproject_toml_parser as pyproject_toml_parser
53
53
  from . import cards as cards
54
54
  from . import client as client
55
55
  from .client.core import namespace as namespace
@@ -152,45 +152,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
152
152
  """
153
153
  ...
154
154
 
155
- @typing.overload
156
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
157
- """
158
- Specifies secrets to be retrieved and injected as environment variables prior to
159
- the execution of a step.
160
-
161
-
162
- Parameters
163
- ----------
164
- sources : List[Union[str, Dict[str, Any]]], default: []
165
- List of secret specs, defining how the secrets are to be retrieved
166
- role : str, optional, default: None
167
- Role to use for fetching secrets
168
- """
169
- ...
170
-
171
- @typing.overload
172
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
173
- ...
174
-
175
- @typing.overload
176
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
177
- ...
178
-
179
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
180
- """
181
- Specifies secrets to be retrieved and injected as environment variables prior to
182
- the execution of a step.
183
-
184
-
185
- Parameters
186
- ----------
187
- sources : List[Union[str, Dict[str, Any]]], default: []
188
- List of secret specs, defining how the secrets are to be retrieved
189
- role : str, optional, default: None
190
- Role to use for fetching secrets
191
- """
192
- ...
193
-
194
155
  @typing.overload
195
156
  def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
196
157
  """
@@ -210,226 +171,215 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
210
171
  """
211
172
  ...
212
173
 
174
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
175
+ """
176
+ Specifies that this step should execute on Kubernetes.
177
+
178
+
179
+ Parameters
180
+ ----------
181
+ cpu : int, default 1
182
+ Number of CPUs required for this step. If `@resources` is
183
+ also present, the maximum value from all decorators is used.
184
+ memory : int, default 4096
185
+ Memory size (in MB) required for this step. If
186
+ `@resources` is also present, the maximum value from all decorators is
187
+ used.
188
+ disk : int, default 10240
189
+ Disk size (in MB) required for this step. If
190
+ `@resources` is also present, the maximum value from all decorators is
191
+ used.
192
+ image : str, optional, default None
193
+ Docker image to use when launching on Kubernetes. If not specified, and
194
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
195
+ not, a default Docker image mapping to the current version of Python is used.
196
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
197
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
198
+ image_pull_secrets: List[str], default []
199
+ The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
200
+ Kubernetes image pull secrets to use when pulling container images
201
+ in Kubernetes.
202
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
203
+ Kubernetes service account to use when launching pod in Kubernetes.
204
+ secrets : List[str], optional, default None
205
+ Kubernetes secrets to use when launching pod in Kubernetes. These
206
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
207
+ in Metaflow configuration.
208
+ node_selector: Union[Dict[str,str], str], optional, default None
209
+ Kubernetes node selector(s) to apply to the pod running the task.
210
+ Can be passed in as a comma separated string of values e.g.
211
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
212
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
213
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
214
+ Kubernetes namespace to use when launching pod in Kubernetes.
215
+ gpu : int, optional, default None
216
+ Number of GPUs required for this step. A value of zero implies that
217
+ the scheduled node should not have GPUs.
218
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
219
+ The vendor of the GPUs to be used for this step.
220
+ tolerations : List[Dict[str,str]], default []
221
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
222
+ Kubernetes tolerations to use when launching pod in Kubernetes.
223
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
224
+ Kubernetes labels to use when launching pod in Kubernetes.
225
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
226
+ Kubernetes annotations to use when launching pod in Kubernetes.
227
+ use_tmpfs : bool, default False
228
+ This enables an explicit tmpfs mount for this step.
229
+ tmpfs_tempdir : bool, default True
230
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
231
+ tmpfs_size : int, optional, default: None
232
+ The value for the size (in MiB) of the tmpfs mount for this step.
233
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
234
+ memory allocated for this step.
235
+ tmpfs_path : str, optional, default /metaflow_temp
236
+ Path to tmpfs mount for this step.
237
+ persistent_volume_claims : Dict[str, str], optional, default None
238
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
239
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
240
+ shared_memory: int, optional
241
+ Shared memory size (in MiB) required for this step
242
+ port: int, optional
243
+ Port number to specify in the Kubernetes job object
244
+ compute_pool : str, optional, default None
245
+ Compute pool to be used for for this step.
246
+ If not specified, any accessible compute pool within the perimeter is used.
247
+ hostname_resolution_timeout: int, default 10 * 60
248
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
249
+ Only applicable when @parallel is used.
250
+ qos: str, default: Burstable
251
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
252
+
253
+ security_context: Dict[str, Any], optional, default None
254
+ Container security context. Applies to the task container. Allows the following keys:
255
+ - privileged: bool, optional, default None
256
+ - allow_privilege_escalation: bool, optional, default None
257
+ - run_as_user: int, optional, default None
258
+ - run_as_group: int, optional, default None
259
+ - run_as_non_root: bool, optional, default None
260
+ """
261
+ ...
262
+
213
263
  @typing.overload
214
- def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
264
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
215
265
  """
216
- Creates a human-readable report, a Metaflow Card, after this step completes.
266
+ Specifies the Conda environment for the step.
217
267
 
218
- Note that you may add multiple `@card` decorators in a step with different parameters.
268
+ Information in this decorator will augment any
269
+ attributes set in the `@conda_base` flow-level decorator. Hence,
270
+ you can use `@conda_base` to set packages required by all
271
+ steps and use `@conda` to specify step-specific overrides.
219
272
 
220
273
 
221
274
  Parameters
222
275
  ----------
223
- type : str, default 'default'
224
- Card type.
225
- id : str, optional, default None
226
- If multiple cards are present, use this id to identify this card.
227
- options : Dict[str, Any], default {}
228
- Options passed to the card. The contents depend on the card type.
229
- timeout : int, default 45
230
- Interrupt reporting if it takes more than this many seconds.
276
+ packages : Dict[str, str], default {}
277
+ Packages to use for this step. The key is the name of the package
278
+ and the value is the version to use.
279
+ libraries : Dict[str, str], default {}
280
+ Supported for backward compatibility. When used with packages, packages will take precedence.
281
+ python : str, optional, default None
282
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
283
+ that the version used will correspond to the version of the Python interpreter used to start the run.
284
+ disabled : bool, default False
285
+ If set to True, disables @conda.
231
286
  """
232
287
  ...
233
288
 
234
289
  @typing.overload
235
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
290
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
236
291
  ...
237
292
 
238
293
  @typing.overload
239
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
294
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
240
295
  ...
241
296
 
242
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
297
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
243
298
  """
244
- Creates a human-readable report, a Metaflow Card, after this step completes.
299
+ Specifies the Conda environment for the step.
245
300
 
246
- Note that you may add multiple `@card` decorators in a step with different parameters.
301
+ Information in this decorator will augment any
302
+ attributes set in the `@conda_base` flow-level decorator. Hence,
303
+ you can use `@conda_base` to set packages required by all
304
+ steps and use `@conda` to specify step-specific overrides.
247
305
 
248
306
 
249
307
  Parameters
250
308
  ----------
251
- type : str, default 'default'
252
- Card type.
253
- id : str, optional, default None
254
- If multiple cards are present, use this id to identify this card.
255
- options : Dict[str, Any], default {}
256
- Options passed to the card. The contents depend on the card type.
257
- timeout : int, default 45
258
- Interrupt reporting if it takes more than this many seconds.
309
+ packages : Dict[str, str], default {}
310
+ Packages to use for this step. The key is the name of the package
311
+ and the value is the version to use.
312
+ libraries : Dict[str, str], default {}
313
+ Supported for backward compatibility. When used with packages, packages will take precedence.
314
+ python : str, optional, default None
315
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
316
+ that the version used will correspond to the version of the Python interpreter used to start the run.
317
+ disabled : bool, default False
318
+ If set to True, disables @conda.
259
319
  """
260
320
  ...
261
321
 
262
322
  @typing.overload
263
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
323
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
264
324
  """
265
- Specifies a timeout for your step.
266
-
267
- This decorator is useful if this step may hang indefinitely.
325
+ Specifies the resources needed when executing this step.
268
326
 
269
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
270
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
271
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
327
+ Use `@resources` to specify the resource requirements
328
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
272
329
 
273
- Note that all the values specified in parameters are added together so if you specify
274
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
330
+ You can choose the compute layer on the command line by executing e.g.
331
+ ```
332
+ python myflow.py run --with batch
333
+ ```
334
+ or
335
+ ```
336
+ python myflow.py run --with kubernetes
337
+ ```
338
+ which executes the flow on the desired system using the
339
+ requirements specified in `@resources`.
275
340
 
276
341
 
277
342
  Parameters
278
343
  ----------
279
- seconds : int, default 0
280
- Number of seconds to wait prior to timing out.
281
- minutes : int, default 0
282
- Number of minutes to wait prior to timing out.
283
- hours : int, default 0
284
- Number of hours to wait prior to timing out.
344
+ cpu : int, default 1
345
+ Number of CPUs required for this step.
346
+ gpu : int, optional, default None
347
+ Number of GPUs required for this step.
348
+ disk : int, optional, default None
349
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
350
+ memory : int, default 4096
351
+ Memory size (in MB) required for this step.
352
+ shared_memory : int, optional, default None
353
+ The value for the size (in MiB) of the /dev/shm volume for this step.
354
+ This parameter maps to the `--shm-size` option in Docker.
285
355
  """
286
356
  ...
287
357
 
288
358
  @typing.overload
289
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
359
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
290
360
  ...
291
361
 
292
362
  @typing.overload
293
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
363
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
294
364
  ...
295
365
 
296
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
366
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
297
367
  """
298
- Specifies a timeout for your step.
368
+ Specifies the resources needed when executing this step.
299
369
 
300
- This decorator is useful if this step may hang indefinitely.
370
+ Use `@resources` to specify the resource requirements
371
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
301
372
 
302
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
303
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
304
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
305
-
306
- Note that all the values specified in parameters are added together so if you specify
307
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
308
-
309
-
310
- Parameters
311
- ----------
312
- seconds : int, default 0
313
- Number of seconds to wait prior to timing out.
314
- minutes : int, default 0
315
- Number of minutes to wait prior to timing out.
316
- hours : int, default 0
317
- Number of hours to wait prior to timing out.
318
- """
319
- ...
320
-
321
- @typing.overload
322
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
323
- """
324
- Specifies that the step will success under all circumstances.
325
-
326
- The decorator will create an optional artifact, specified by `var`, which
327
- contains the exception raised. You can use it to detect the presence
328
- of errors, indicating that all happy-path artifacts produced by the step
329
- are missing.
330
-
331
-
332
- Parameters
333
- ----------
334
- var : str, optional, default None
335
- Name of the artifact in which to store the caught exception.
336
- If not specified, the exception is not stored.
337
- print_exception : bool, default True
338
- Determines whether or not the exception is printed to
339
- stdout when caught.
340
- """
341
- ...
342
-
343
- @typing.overload
344
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
345
- ...
346
-
347
- @typing.overload
348
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
349
- ...
350
-
351
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
352
- """
353
- Specifies that the step will success under all circumstances.
354
-
355
- The decorator will create an optional artifact, specified by `var`, which
356
- contains the exception raised. You can use it to detect the presence
357
- of errors, indicating that all happy-path artifacts produced by the step
358
- are missing.
359
-
360
-
361
- Parameters
362
- ----------
363
- var : str, optional, default None
364
- Name of the artifact in which to store the caught exception.
365
- If not specified, the exception is not stored.
366
- print_exception : bool, default True
367
- Determines whether or not the exception is printed to
368
- stdout when caught.
369
- """
370
- ...
371
-
372
- @typing.overload
373
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
374
- """
375
- Specifies the resources needed when executing this step.
376
-
377
- Use `@resources` to specify the resource requirements
378
- independently of the specific compute layer (`@batch`, `@kubernetes`).
379
-
380
- You can choose the compute layer on the command line by executing e.g.
381
- ```
382
- python myflow.py run --with batch
383
- ```
384
- or
385
- ```
386
- python myflow.py run --with kubernetes
387
- ```
388
- which executes the flow on the desired system using the
389
- requirements specified in `@resources`.
390
-
391
-
392
- Parameters
393
- ----------
394
- cpu : int, default 1
395
- Number of CPUs required for this step.
396
- gpu : int, optional, default None
397
- Number of GPUs required for this step.
398
- disk : int, optional, default None
399
- Disk size (in MB) required for this step. Only applies on Kubernetes.
400
- memory : int, default 4096
401
- Memory size (in MB) required for this step.
402
- shared_memory : int, optional, default None
403
- The value for the size (in MiB) of the /dev/shm volume for this step.
404
- This parameter maps to the `--shm-size` option in Docker.
405
- """
406
- ...
407
-
408
- @typing.overload
409
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
410
- ...
411
-
412
- @typing.overload
413
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
414
- ...
415
-
416
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
417
- """
418
- Specifies the resources needed when executing this step.
419
-
420
- Use `@resources` to specify the resource requirements
421
- independently of the specific compute layer (`@batch`, `@kubernetes`).
422
-
423
- You can choose the compute layer on the command line by executing e.g.
424
- ```
425
- python myflow.py run --with batch
426
- ```
427
- or
428
- ```
429
- python myflow.py run --with kubernetes
430
- ```
431
- which executes the flow on the desired system using the
432
- requirements specified in `@resources`.
373
+ You can choose the compute layer on the command line by executing e.g.
374
+ ```
375
+ python myflow.py run --with batch
376
+ ```
377
+ or
378
+ ```
379
+ python myflow.py run --with kubernetes
380
+ ```
381
+ which executes the flow on the desired system using the
382
+ requirements specified in `@resources`.
433
383
 
434
384
 
435
385
  Parameters
@@ -448,65 +398,6 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
448
398
  """
449
399
  ...
450
400
 
451
- @typing.overload
452
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
453
- """
454
- Specifies the Conda environment for the step.
455
-
456
- Information in this decorator will augment any
457
- attributes set in the `@conda_base` flow-level decorator. Hence,
458
- you can use `@conda_base` to set packages required by all
459
- steps and use `@conda` to specify step-specific overrides.
460
-
461
-
462
- Parameters
463
- ----------
464
- packages : Dict[str, str], default {}
465
- Packages to use for this step. The key is the name of the package
466
- and the value is the version to use.
467
- libraries : Dict[str, str], default {}
468
- Supported for backward compatibility. When used with packages, packages will take precedence.
469
- python : str, optional, default None
470
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
471
- that the version used will correspond to the version of the Python interpreter used to start the run.
472
- disabled : bool, default False
473
- If set to True, disables @conda.
474
- """
475
- ...
476
-
477
- @typing.overload
478
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
479
- ...
480
-
481
- @typing.overload
482
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
483
- ...
484
-
485
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
486
- """
487
- Specifies the Conda environment for the step.
488
-
489
- Information in this decorator will augment any
490
- attributes set in the `@conda_base` flow-level decorator. Hence,
491
- you can use `@conda_base` to set packages required by all
492
- steps and use `@conda` to specify step-specific overrides.
493
-
494
-
495
- Parameters
496
- ----------
497
- packages : Dict[str, str], default {}
498
- Packages to use for this step. The key is the name of the package
499
- and the value is the version to use.
500
- libraries : Dict[str, str], default {}
501
- Supported for backward compatibility. When used with packages, packages will take precedence.
502
- python : str, optional, default None
503
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
504
- that the version used will correspond to the version of the Python interpreter used to start the run.
505
- disabled : bool, default False
506
- If set to True, disables @conda.
507
- """
508
- ...
509
-
510
401
  @typing.overload
511
402
  def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
512
403
  """
@@ -563,182 +454,58 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
563
454
  ...
564
455
 
565
456
  @typing.overload
566
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
457
+ def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
567
458
  """
568
- Specifies the PyPI packages for the step.
459
+ Creates a human-readable report, a Metaflow Card, after this step completes.
569
460
 
570
- Information in this decorator will augment any
571
- attributes set in the `@pyi_base` flow-level decorator. Hence,
572
- you can use `@pypi_base` to set packages required by all
573
- steps and use `@pypi` to specify step-specific overrides.
461
+ Note that you may add multiple `@card` decorators in a step with different parameters.
574
462
 
575
463
 
576
464
  Parameters
577
465
  ----------
578
- packages : Dict[str, str], default: {}
579
- Packages to use for this step. The key is the name of the package
580
- and the value is the version to use.
581
- python : str, optional, default: None
582
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
583
- that the version used will correspond to the version of the Python interpreter used to start the run.
466
+ type : str, default 'default'
467
+ Card type.
468
+ id : str, optional, default None
469
+ If multiple cards are present, use this id to identify this card.
470
+ options : Dict[str, Any], default {}
471
+ Options passed to the card. The contents depend on the card type.
472
+ timeout : int, default 45
473
+ Interrupt reporting if it takes more than this many seconds.
584
474
  """
585
475
  ...
586
476
 
587
477
  @typing.overload
588
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
478
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
589
479
  ...
590
480
 
591
481
  @typing.overload
592
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
482
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
593
483
  ...
594
484
 
595
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
485
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
596
486
  """
597
- Specifies the PyPI packages for the step.
487
+ Creates a human-readable report, a Metaflow Card, after this step completes.
598
488
 
599
- Information in this decorator will augment any
600
- attributes set in the `@pyi_base` flow-level decorator. Hence,
601
- you can use `@pypi_base` to set packages required by all
602
- steps and use `@pypi` to specify step-specific overrides.
489
+ Note that you may add multiple `@card` decorators in a step with different parameters.
603
490
 
604
491
 
605
492
  Parameters
606
493
  ----------
607
- packages : Dict[str, str], default: {}
608
- Packages to use for this step. The key is the name of the package
609
- and the value is the version to use.
610
- python : str, optional, default: None
611
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
612
- that the version used will correspond to the version of the Python interpreter used to start the run.
494
+ type : str, default 'default'
495
+ Card type.
496
+ id : str, optional, default None
497
+ If multiple cards are present, use this id to identify this card.
498
+ options : Dict[str, Any], default {}
499
+ Options passed to the card. The contents depend on the card type.
500
+ timeout : int, default 45
501
+ Interrupt reporting if it takes more than this many seconds.
613
502
  """
614
503
  ...
615
504
 
616
505
  @typing.overload
617
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
506
+ def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
618
507
  """
619
- Specifies environment variables to be set prior to the execution of a step.
620
-
621
-
622
- Parameters
623
- ----------
624
- vars : Dict[str, str], default {}
625
- Dictionary of environment variables to set.
626
- """
627
- ...
628
-
629
- @typing.overload
630
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
631
- ...
632
-
633
- @typing.overload
634
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
635
- ...
636
-
637
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
638
- """
639
- Specifies environment variables to be set prior to the execution of a step.
640
-
641
-
642
- Parameters
643
- ----------
644
- vars : Dict[str, str], default {}
645
- Dictionary of environment variables to set.
646
- """
647
- ...
648
-
649
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', image_pull_secrets: typing.List[str] = [], service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[typing.Dict[str, str]] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable', security_context: typing.Optional[typing.Dict[str, typing.Any]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
650
- """
651
- Specifies that this step should execute on Kubernetes.
652
-
653
-
654
- Parameters
655
- ----------
656
- cpu : int, default 1
657
- Number of CPUs required for this step. If `@resources` is
658
- also present, the maximum value from all decorators is used.
659
- memory : int, default 4096
660
- Memory size (in MB) required for this step. If
661
- `@resources` is also present, the maximum value from all decorators is
662
- used.
663
- disk : int, default 10240
664
- Disk size (in MB) required for this step. If
665
- `@resources` is also present, the maximum value from all decorators is
666
- used.
667
- image : str, optional, default None
668
- Docker image to use when launching on Kubernetes. If not specified, and
669
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
670
- not, a default Docker image mapping to the current version of Python is used.
671
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
672
- If given, the imagePullPolicy to be applied to the Docker image of the step.
673
- image_pull_secrets: List[str], default []
674
- The default is extracted from METAFLOW_KUBERNETES_IMAGE_PULL_SECRETS.
675
- Kubernetes image pull secrets to use when pulling container images
676
- in Kubernetes.
677
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
678
- Kubernetes service account to use when launching pod in Kubernetes.
679
- secrets : List[str], optional, default None
680
- Kubernetes secrets to use when launching pod in Kubernetes. These
681
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
682
- in Metaflow configuration.
683
- node_selector: Union[Dict[str,str], str], optional, default None
684
- Kubernetes node selector(s) to apply to the pod running the task.
685
- Can be passed in as a comma separated string of values e.g.
686
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
687
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
688
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
689
- Kubernetes namespace to use when launching pod in Kubernetes.
690
- gpu : int, optional, default None
691
- Number of GPUs required for this step. A value of zero implies that
692
- the scheduled node should not have GPUs.
693
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
694
- The vendor of the GPUs to be used for this step.
695
- tolerations : List[Dict[str,str]], default []
696
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
697
- Kubernetes tolerations to use when launching pod in Kubernetes.
698
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
699
- Kubernetes labels to use when launching pod in Kubernetes.
700
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
701
- Kubernetes annotations to use when launching pod in Kubernetes.
702
- use_tmpfs : bool, default False
703
- This enables an explicit tmpfs mount for this step.
704
- tmpfs_tempdir : bool, default True
705
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
706
- tmpfs_size : int, optional, default: None
707
- The value for the size (in MiB) of the tmpfs mount for this step.
708
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
709
- memory allocated for this step.
710
- tmpfs_path : str, optional, default /metaflow_temp
711
- Path to tmpfs mount for this step.
712
- persistent_volume_claims : Dict[str, str], optional, default None
713
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
714
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
715
- shared_memory: int, optional
716
- Shared memory size (in MiB) required for this step
717
- port: int, optional
718
- Port number to specify in the Kubernetes job object
719
- compute_pool : str, optional, default None
720
- Compute pool to be used for for this step.
721
- If not specified, any accessible compute pool within the perimeter is used.
722
- hostname_resolution_timeout: int, default 10 * 60
723
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
724
- Only applicable when @parallel is used.
725
- qos: str, default: Burstable
726
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
727
-
728
- security_context: Dict[str, Any], optional, default None
729
- Container security context. Applies to the task container. Allows the following keys:
730
- - privileged: bool, optional, default None
731
- - allow_privilege_escalation: bool, optional, default None
732
- - run_as_user: int, optional, default None
733
- - run_as_group: int, optional, default None
734
- - run_as_non_root: bool, optional, default None
735
- """
736
- ...
737
-
738
- @typing.overload
739
- def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = 'METAFLOW_BATCH_JOB_QUEUE', iam_role: str = 'METAFLOW_ECS_S3_ACCESS_IAM_ROLE', execution_role: str = 'METAFLOW_ECS_FARGATE_EXECUTION_ROLE', shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
740
- """
741
- Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
508
+ Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
742
509
 
743
510
 
744
511
  Parameters
@@ -885,140 +652,242 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
885
652
  ...
886
653
 
887
654
  @typing.overload
888
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
655
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
889
656
  """
890
- Specifies the event(s) that this flow depends on.
657
+ Specifies a timeout for your step.
891
658
 
892
- ```
893
- @trigger(event='foo')
894
- ```
895
- or
896
- ```
897
- @trigger(events=['foo', 'bar'])
898
- ```
659
+ This decorator is useful if this step may hang indefinitely.
899
660
 
900
- Additionally, you can specify the parameter mappings
901
- to map event payload to Metaflow parameters for the flow.
902
- ```
903
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
904
- ```
905
- or
906
- ```
907
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
908
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
909
- ```
661
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
662
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
663
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
910
664
 
911
- 'parameters' can also be a list of strings and tuples like so:
912
- ```
913
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
914
- ```
915
- This is equivalent to:
916
- ```
917
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
918
- ```
665
+ Note that all the values specified in parameters are added together so if you specify
666
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
919
667
 
920
668
 
921
669
  Parameters
922
670
  ----------
923
- event : Union[str, Dict[str, Any]], optional, default None
924
- Event dependency for this flow.
925
- events : List[Union[str, Dict[str, Any]]], default []
926
- Events dependency for this flow.
927
- options : Dict[str, Any], default {}
928
- Backend-specific configuration for tuning eventing behavior.
671
+ seconds : int, default 0
672
+ Number of seconds to wait prior to timing out.
673
+ minutes : int, default 0
674
+ Number of minutes to wait prior to timing out.
675
+ hours : int, default 0
676
+ Number of hours to wait prior to timing out.
929
677
  """
930
678
  ...
931
679
 
932
680
  @typing.overload
933
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
681
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
934
682
  ...
935
683
 
936
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
684
+ @typing.overload
685
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
686
+ ...
687
+
688
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
937
689
  """
938
- Specifies the event(s) that this flow depends on.
690
+ Specifies a timeout for your step.
939
691
 
940
- ```
941
- @trigger(event='foo')
942
- ```
943
- or
944
- ```
945
- @trigger(events=['foo', 'bar'])
946
- ```
692
+ This decorator is useful if this step may hang indefinitely.
947
693
 
948
- Additionally, you can specify the parameter mappings
949
- to map event payload to Metaflow parameters for the flow.
950
- ```
951
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
952
- ```
953
- or
954
- ```
955
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
956
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
957
- ```
694
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
695
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
696
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
958
697
 
959
- 'parameters' can also be a list of strings and tuples like so:
960
- ```
961
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
962
- ```
963
- This is equivalent to:
964
- ```
965
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
966
- ```
698
+ Note that all the values specified in parameters are added together so if you specify
699
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
967
700
 
968
701
 
969
702
  Parameters
970
703
  ----------
971
- event : Union[str, Dict[str, Any]], optional, default None
972
- Event dependency for this flow.
973
- events : List[Union[str, Dict[str, Any]]], default []
974
- Events dependency for this flow.
975
- options : Dict[str, Any], default {}
976
- Backend-specific configuration for tuning eventing behavior.
704
+ seconds : int, default 0
705
+ Number of seconds to wait prior to timing out.
706
+ minutes : int, default 0
707
+ Number of minutes to wait prior to timing out.
708
+ hours : int, default 0
709
+ Number of hours to wait prior to timing out.
977
710
  """
978
711
  ...
979
712
 
980
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
713
+ @typing.overload
714
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
981
715
  """
982
- Specifies what flows belong to the same project.
716
+ Specifies secrets to be retrieved and injected as environment variables prior to
717
+ the execution of a step.
983
718
 
984
- A project-specific namespace is created for all flows that
985
- use the same `@project(name)`.
719
+
720
+ Parameters
721
+ ----------
722
+ sources : List[Union[str, Dict[str, Any]]], default: []
723
+ List of secret specs, defining how the secrets are to be retrieved
724
+ role : str, optional, default: None
725
+ Role to use for fetching secrets
726
+ """
727
+ ...
728
+
729
+ @typing.overload
730
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
731
+ ...
732
+
733
+ @typing.overload
734
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
735
+ ...
736
+
737
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], role: typing.Optional[str] = None):
738
+ """
739
+ Specifies secrets to be retrieved and injected as environment variables prior to
740
+ the execution of a step.
986
741
 
987
742
 
988
743
  Parameters
989
744
  ----------
990
- name : str
991
- Project name. Make sure that the name is unique amongst all
992
- projects that use the same production scheduler. The name may
993
- contain only lowercase alphanumeric characters and underscores.
745
+ sources : List[Union[str, Dict[str, Any]]], default: []
746
+ List of secret specs, defining how the secrets are to be retrieved
747
+ role : str, optional, default: None
748
+ Role to use for fetching secrets
749
+ """
750
+ ...
751
+
752
+ @typing.overload
753
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
754
+ """
755
+ Specifies the PyPI packages for the step.
994
756
 
995
- branch : Optional[str], default None
996
- The branch to use. If not specified, the branch is set to
997
- `user.<username>` unless `production` is set to `True`. This can
998
- also be set on the command line using `--branch` as a top-level option.
999
- It is an error to specify `branch` in the decorator and on the command line.
757
+ Information in this decorator will augment any
758
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
759
+ you can use `@pypi_base` to set packages required by all
760
+ steps and use `@pypi` to specify step-specific overrides.
1000
761
 
1001
- production : bool, default False
1002
- Whether or not the branch is the production branch. This can also be set on the
1003
- command line using `--production` as a top-level option. It is an error to specify
1004
- `production` in the decorator and on the command line.
1005
- The project branch name will be:
1006
- - if `branch` is specified:
1007
- - if `production` is True: `prod.<branch>`
1008
- - if `production` is False: `test.<branch>`
1009
- - if `branch` is not specified:
1010
- - if `production` is True: `prod`
1011
- - if `production` is False: `user.<username>`
762
+
763
+ Parameters
764
+ ----------
765
+ packages : Dict[str, str], default: {}
766
+ Packages to use for this step. The key is the name of the package
767
+ and the value is the version to use.
768
+ python : str, optional, default: None
769
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
770
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1012
771
  """
1013
772
  ...
1014
773
 
1015
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
774
+ @typing.overload
775
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
776
+ ...
777
+
778
+ @typing.overload
779
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
780
+ ...
781
+
782
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1016
783
  """
1017
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1018
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1019
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1020
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1021
- starts only after all sensors finish.
784
+ Specifies the PyPI packages for the step.
785
+
786
+ Information in this decorator will augment any
787
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
788
+ you can use `@pypi_base` to set packages required by all
789
+ steps and use `@pypi` to specify step-specific overrides.
790
+
791
+
792
+ Parameters
793
+ ----------
794
+ packages : Dict[str, str], default: {}
795
+ Packages to use for this step. The key is the name of the package
796
+ and the value is the version to use.
797
+ python : str, optional, default: None
798
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
799
+ that the version used will correspond to the version of the Python interpreter used to start the run.
800
+ """
801
+ ...
802
+
803
+ @typing.overload
804
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
805
+ """
806
+ Specifies environment variables to be set prior to the execution of a step.
807
+
808
+
809
+ Parameters
810
+ ----------
811
+ vars : Dict[str, str], default {}
812
+ Dictionary of environment variables to set.
813
+ """
814
+ ...
815
+
816
+ @typing.overload
817
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
818
+ ...
819
+
820
+ @typing.overload
821
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
822
+ ...
823
+
824
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
825
+ """
826
+ Specifies environment variables to be set prior to the execution of a step.
827
+
828
+
829
+ Parameters
830
+ ----------
831
+ vars : Dict[str, str], default {}
832
+ Dictionary of environment variables to set.
833
+ """
834
+ ...
835
+
836
+ @typing.overload
837
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
838
+ """
839
+ Specifies that the step will success under all circumstances.
840
+
841
+ The decorator will create an optional artifact, specified by `var`, which
842
+ contains the exception raised. You can use it to detect the presence
843
+ of errors, indicating that all happy-path artifacts produced by the step
844
+ are missing.
845
+
846
+
847
+ Parameters
848
+ ----------
849
+ var : str, optional, default None
850
+ Name of the artifact in which to store the caught exception.
851
+ If not specified, the exception is not stored.
852
+ print_exception : bool, default True
853
+ Determines whether or not the exception is printed to
854
+ stdout when caught.
855
+ """
856
+ ...
857
+
858
+ @typing.overload
859
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
860
+ ...
861
+
862
+ @typing.overload
863
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
864
+ ...
865
+
866
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
867
+ """
868
+ Specifies that the step will success under all circumstances.
869
+
870
+ The decorator will create an optional artifact, specified by `var`, which
871
+ contains the exception raised. You can use it to detect the presence
872
+ of errors, indicating that all happy-path artifacts produced by the step
873
+ are missing.
874
+
875
+
876
+ Parameters
877
+ ----------
878
+ var : str, optional, default None
879
+ Name of the artifact in which to store the caught exception.
880
+ If not specified, the exception is not stored.
881
+ print_exception : bool, default True
882
+ Determines whether or not the exception is printed to
883
+ stdout when caught.
884
+ """
885
+ ...
886
+
887
+ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
888
+ """
889
+ The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
890
+ This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1022
891
 
1023
892
 
1024
893
  Parameters
@@ -1040,18 +909,21 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1040
909
  Name of the sensor on Airflow
1041
910
  description : str
1042
911
  Description of sensor in the Airflow UI
1043
- bucket_key : Union[str, List[str]]
1044
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1045
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1046
- bucket_name : str
1047
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1048
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1049
- wildcard_match : bool
1050
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1051
- aws_conn_id : str
1052
- a reference to the s3 connection on Airflow. (Default: None)
1053
- verify : bool
1054
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
912
+ external_dag_id : str
913
+ The dag_id that contains the task you want to wait for.
914
+ external_task_ids : List[str]
915
+ The list of task_ids that you want to wait for.
916
+ If None (default value) the sensor waits for the DAG. (Default: None)
917
+ allowed_states : List[str]
918
+ Iterable of allowed states, (Default: ['success'])
919
+ failed_states : List[str]
920
+ Iterable of failed or dis-allowed states. (Default: None)
921
+ execution_delta : datetime.timedelta
922
+ time difference with the previous execution to look at,
923
+ the default is the same logical date as the current task or DAG. (Default: None)
924
+ check_existence: bool
925
+ Set to True to check if the external task exists or check if
926
+ the DAG to wait for exists. (Default: True)
1055
927
  """
1056
928
  ...
1057
929
 
@@ -1068,38 +940,127 @@ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[s
1068
940
  packages : Dict[str, str], default: {}
1069
941
  Packages to use for this flow. The key is the name of the package
1070
942
  and the value is the version to use.
1071
- python : str, optional, default: None
943
+ python : str, optional, default: None
944
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
945
+ that the version used will correspond to the version of the Python interpreter used to start the run.
946
+ """
947
+ ...
948
+
949
+ @typing.overload
950
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
951
+ ...
952
+
953
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
954
+ """
955
+ Specifies the PyPI packages for all steps of the flow.
956
+
957
+ Use `@pypi_base` to set common packages required by all
958
+ steps and use `@pypi` to specify step-specific overrides.
959
+
960
+ Parameters
961
+ ----------
962
+ packages : Dict[str, str], default: {}
963
+ Packages to use for this flow. The key is the name of the package
964
+ and the value is the version to use.
965
+ python : str, optional, default: None
966
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
967
+ that the version used will correspond to the version of the Python interpreter used to start the run.
968
+ """
969
+ ...
970
+
971
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
972
+ """
973
+ Specifies what flows belong to the same project.
974
+
975
+ A project-specific namespace is created for all flows that
976
+ use the same `@project(name)`.
977
+
978
+
979
+ Parameters
980
+ ----------
981
+ name : str
982
+ Project name. Make sure that the name is unique amongst all
983
+ projects that use the same production scheduler. The name may
984
+ contain only lowercase alphanumeric characters and underscores.
985
+
986
+ branch : Optional[str], default None
987
+ The branch to use. If not specified, the branch is set to
988
+ `user.<username>` unless `production` is set to `True`. This can
989
+ also be set on the command line using `--branch` as a top-level option.
990
+ It is an error to specify `branch` in the decorator and on the command line.
991
+
992
+ production : bool, default False
993
+ Whether or not the branch is the production branch. This can also be set on the
994
+ command line using `--production` as a top-level option. It is an error to specify
995
+ `production` in the decorator and on the command line.
996
+ The project branch name will be:
997
+ - if `branch` is specified:
998
+ - if `production` is True: `prod.<branch>`
999
+ - if `production` is False: `test.<branch>`
1000
+ - if `branch` is not specified:
1001
+ - if `production` is True: `prod`
1002
+ - if `production` is False: `user.<username>`
1003
+ """
1004
+ ...
1005
+
1006
+ @typing.overload
1007
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1008
+ """
1009
+ Specifies the Conda environment for all steps of the flow.
1010
+
1011
+ Use `@conda_base` to set common libraries required by all
1012
+ steps and use `@conda` to specify step-specific additions.
1013
+
1014
+
1015
+ Parameters
1016
+ ----------
1017
+ packages : Dict[str, str], default {}
1018
+ Packages to use for this flow. The key is the name of the package
1019
+ and the value is the version to use.
1020
+ libraries : Dict[str, str], default {}
1021
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1022
+ python : str, optional, default None
1072
1023
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1073
1024
  that the version used will correspond to the version of the Python interpreter used to start the run.
1025
+ disabled : bool, default False
1026
+ If set to True, disables Conda.
1074
1027
  """
1075
1028
  ...
1076
1029
 
1077
1030
  @typing.overload
1078
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1031
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1079
1032
  ...
1080
1033
 
1081
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1034
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1082
1035
  """
1083
- Specifies the PyPI packages for all steps of the flow.
1036
+ Specifies the Conda environment for all steps of the flow.
1037
+
1038
+ Use `@conda_base` to set common libraries required by all
1039
+ steps and use `@conda` to specify step-specific additions.
1084
1040
 
1085
- Use `@pypi_base` to set common packages required by all
1086
- steps and use `@pypi` to specify step-specific overrides.
1087
1041
 
1088
1042
  Parameters
1089
1043
  ----------
1090
- packages : Dict[str, str], default: {}
1044
+ packages : Dict[str, str], default {}
1091
1045
  Packages to use for this flow. The key is the name of the package
1092
1046
  and the value is the version to use.
1093
- python : str, optional, default: None
1047
+ libraries : Dict[str, str], default {}
1048
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1049
+ python : str, optional, default None
1094
1050
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1095
1051
  that the version used will correspond to the version of the Python interpreter used to start the run.
1052
+ disabled : bool, default False
1053
+ If set to True, disables Conda.
1096
1054
  """
1097
1055
  ...
1098
1056
 
1099
- def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1057
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1100
1058
  """
1101
- The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
1102
- This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
1059
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1060
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1061
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1062
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1063
+ starts only after all sensors finish.
1103
1064
 
1104
1065
 
1105
1066
  Parameters
@@ -1121,120 +1082,109 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1121
1082
  Name of the sensor on Airflow
1122
1083
  description : str
1123
1084
  Description of sensor in the Airflow UI
1124
- external_dag_id : str
1125
- The dag_id that contains the task you want to wait for.
1126
- external_task_ids : List[str]
1127
- The list of task_ids that you want to wait for.
1128
- If None (default value) the sensor waits for the DAG. (Default: None)
1129
- allowed_states : List[str]
1130
- Iterable of allowed states, (Default: ['success'])
1131
- failed_states : List[str]
1132
- Iterable of failed or dis-allowed states. (Default: None)
1133
- execution_delta : datetime.timedelta
1134
- time difference with the previous execution to look at,
1135
- the default is the same logical date as the current task or DAG. (Default: None)
1136
- check_existence: bool
1137
- Set to True to check if the external task exists or check if
1138
- the DAG to wait for exists. (Default: True)
1085
+ bucket_key : Union[str, List[str]]
1086
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1087
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1088
+ bucket_name : str
1089
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1090
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1091
+ wildcard_match : bool
1092
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1093
+ aws_conn_id : str
1094
+ a reference to the s3 connection on Airflow. (Default: None)
1095
+ verify : bool
1096
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1139
1097
  """
1140
1098
  ...
1141
1099
 
1142
1100
  @typing.overload
1143
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1101
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1144
1102
  """
1145
- Specifies the flow(s) that this flow depends on.
1103
+ Specifies the event(s) that this flow depends on.
1146
1104
 
1147
1105
  ```
1148
- @trigger_on_finish(flow='FooFlow')
1106
+ @trigger(event='foo')
1149
1107
  ```
1150
1108
  or
1151
1109
  ```
1152
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1110
+ @trigger(events=['foo', 'bar'])
1153
1111
  ```
1154
- This decorator respects the @project decorator and triggers the flow
1155
- when upstream runs within the same namespace complete successfully
1156
1112
 
1157
- Additionally, you can specify project aware upstream flow dependencies
1158
- by specifying the fully qualified project_flow_name.
1113
+ Additionally, you can specify the parameter mappings
1114
+ to map event payload to Metaflow parameters for the flow.
1159
1115
  ```
1160
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1116
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1161
1117
  ```
1162
1118
  or
1163
1119
  ```
1164
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1120
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1121
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1165
1122
  ```
1166
1123
 
1167
- You can also specify just the project or project branch (other values will be
1168
- inferred from the current project or project branch):
1124
+ 'parameters' can also be a list of strings and tuples like so:
1169
1125
  ```
1170
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1126
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1127
+ ```
1128
+ This is equivalent to:
1129
+ ```
1130
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1171
1131
  ```
1172
-
1173
- Note that `branch` is typically one of:
1174
- - `prod`
1175
- - `user.bob`
1176
- - `test.my_experiment`
1177
- - `prod.staging`
1178
1132
 
1179
1133
 
1180
1134
  Parameters
1181
1135
  ----------
1182
- flow : Union[str, Dict[str, str]], optional, default None
1183
- Upstream flow dependency for this flow.
1184
- flows : List[Union[str, Dict[str, str]]], default []
1185
- Upstream flow dependencies for this flow.
1136
+ event : Union[str, Dict[str, Any]], optional, default None
1137
+ Event dependency for this flow.
1138
+ events : List[Union[str, Dict[str, Any]]], default []
1139
+ Events dependency for this flow.
1186
1140
  options : Dict[str, Any], default {}
1187
1141
  Backend-specific configuration for tuning eventing behavior.
1188
1142
  """
1189
1143
  ...
1190
1144
 
1191
1145
  @typing.overload
1192
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1146
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1193
1147
  ...
1194
1148
 
1195
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1149
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1196
1150
  """
1197
- Specifies the flow(s) that this flow depends on.
1151
+ Specifies the event(s) that this flow depends on.
1198
1152
 
1199
1153
  ```
1200
- @trigger_on_finish(flow='FooFlow')
1154
+ @trigger(event='foo')
1201
1155
  ```
1202
1156
  or
1203
1157
  ```
1204
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1158
+ @trigger(events=['foo', 'bar'])
1205
1159
  ```
1206
- This decorator respects the @project decorator and triggers the flow
1207
- when upstream runs within the same namespace complete successfully
1208
1160
 
1209
- Additionally, you can specify project aware upstream flow dependencies
1210
- by specifying the fully qualified project_flow_name.
1161
+ Additionally, you can specify the parameter mappings
1162
+ to map event payload to Metaflow parameters for the flow.
1211
1163
  ```
1212
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1164
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1213
1165
  ```
1214
1166
  or
1215
1167
  ```
1216
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1168
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1169
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1217
1170
  ```
1218
1171
 
1219
- You can also specify just the project or project branch (other values will be
1220
- inferred from the current project or project branch):
1172
+ 'parameters' can also be a list of strings and tuples like so:
1221
1173
  ```
1222
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1174
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1175
+ ```
1176
+ This is equivalent to:
1177
+ ```
1178
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1223
1179
  ```
1224
-
1225
- Note that `branch` is typically one of:
1226
- - `prod`
1227
- - `user.bob`
1228
- - `test.my_experiment`
1229
- - `prod.staging`
1230
1180
 
1231
1181
 
1232
1182
  Parameters
1233
1183
  ----------
1234
- flow : Union[str, Dict[str, str]], optional, default None
1235
- Upstream flow dependency for this flow.
1236
- flows : List[Union[str, Dict[str, str]]], default []
1237
- Upstream flow dependencies for this flow.
1184
+ event : Union[str, Dict[str, Any]], optional, default None
1185
+ Event dependency for this flow.
1186
+ events : List[Union[str, Dict[str, Any]]], default []
1187
+ Events dependency for this flow.
1238
1188
  options : Dict[str, Any], default {}
1239
1189
  Backend-specific configuration for tuning eventing behavior.
1240
1190
  """
@@ -1292,53 +1242,103 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
1292
1242
  ...
1293
1243
 
1294
1244
  @typing.overload
1295
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1245
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1296
1246
  """
1297
- Specifies the Conda environment for all steps of the flow.
1247
+ Specifies the flow(s) that this flow depends on.
1298
1248
 
1299
- Use `@conda_base` to set common libraries required by all
1300
- steps and use `@conda` to specify step-specific additions.
1249
+ ```
1250
+ @trigger_on_finish(flow='FooFlow')
1251
+ ```
1252
+ or
1253
+ ```
1254
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1255
+ ```
1256
+ This decorator respects the @project decorator and triggers the flow
1257
+ when upstream runs within the same namespace complete successfully
1258
+
1259
+ Additionally, you can specify project aware upstream flow dependencies
1260
+ by specifying the fully qualified project_flow_name.
1261
+ ```
1262
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1263
+ ```
1264
+ or
1265
+ ```
1266
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1267
+ ```
1268
+
1269
+ You can also specify just the project or project branch (other values will be
1270
+ inferred from the current project or project branch):
1271
+ ```
1272
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1273
+ ```
1274
+
1275
+ Note that `branch` is typically one of:
1276
+ - `prod`
1277
+ - `user.bob`
1278
+ - `test.my_experiment`
1279
+ - `prod.staging`
1301
1280
 
1302
1281
 
1303
1282
  Parameters
1304
1283
  ----------
1305
- packages : Dict[str, str], default {}
1306
- Packages to use for this flow. The key is the name of the package
1307
- and the value is the version to use.
1308
- libraries : Dict[str, str], default {}
1309
- Supported for backward compatibility. When used with packages, packages will take precedence.
1310
- python : str, optional, default None
1311
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1312
- that the version used will correspond to the version of the Python interpreter used to start the run.
1313
- disabled : bool, default False
1314
- If set to True, disables Conda.
1284
+ flow : Union[str, Dict[str, str]], optional, default None
1285
+ Upstream flow dependency for this flow.
1286
+ flows : List[Union[str, Dict[str, str]]], default []
1287
+ Upstream flow dependencies for this flow.
1288
+ options : Dict[str, Any], default {}
1289
+ Backend-specific configuration for tuning eventing behavior.
1315
1290
  """
1316
1291
  ...
1317
1292
 
1318
1293
  @typing.overload
1319
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1294
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1320
1295
  ...
1321
1296
 
1322
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1297
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1323
1298
  """
1324
- Specifies the Conda environment for all steps of the flow.
1299
+ Specifies the flow(s) that this flow depends on.
1325
1300
 
1326
- Use `@conda_base` to set common libraries required by all
1327
- steps and use `@conda` to specify step-specific additions.
1301
+ ```
1302
+ @trigger_on_finish(flow='FooFlow')
1303
+ ```
1304
+ or
1305
+ ```
1306
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1307
+ ```
1308
+ This decorator respects the @project decorator and triggers the flow
1309
+ when upstream runs within the same namespace complete successfully
1310
+
1311
+ Additionally, you can specify project aware upstream flow dependencies
1312
+ by specifying the fully qualified project_flow_name.
1313
+ ```
1314
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1315
+ ```
1316
+ or
1317
+ ```
1318
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1319
+ ```
1320
+
1321
+ You can also specify just the project or project branch (other values will be
1322
+ inferred from the current project or project branch):
1323
+ ```
1324
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1325
+ ```
1326
+
1327
+ Note that `branch` is typically one of:
1328
+ - `prod`
1329
+ - `user.bob`
1330
+ - `test.my_experiment`
1331
+ - `prod.staging`
1328
1332
 
1329
1333
 
1330
1334
  Parameters
1331
1335
  ----------
1332
- packages : Dict[str, str], default {}
1333
- Packages to use for this flow. The key is the name of the package
1334
- and the value is the version to use.
1335
- libraries : Dict[str, str], default {}
1336
- Supported for backward compatibility. When used with packages, packages will take precedence.
1337
- python : str, optional, default None
1338
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1339
- that the version used will correspond to the version of the Python interpreter used to start the run.
1340
- disabled : bool, default False
1341
- If set to True, disables Conda.
1336
+ flow : Union[str, Dict[str, str]], optional, default None
1337
+ Upstream flow dependency for this flow.
1338
+ flows : List[Union[str, Dict[str, str]]], default []
1339
+ Upstream flow dependencies for this flow.
1340
+ options : Dict[str, Any], default {}
1341
+ Backend-specific configuration for tuning eventing behavior.
1342
1342
  """
1343
1343
  ...
1344
1344