metaflow-stubs 2.14.0__py2.py3-none-any.whl → 2.14.2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. metaflow-stubs/__init__.pyi +493 -493
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -7
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +93 -4
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +2 -2
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +26 -2
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +3 -5
  20. metaflow-stubs/metaflow_current.pyi +5 -5
  21. metaflow-stubs/multicore_utils.pyi +2 -2
  22. metaflow-stubs/parameters.pyi +2 -2
  23. metaflow-stubs/plugins/__init__.pyi +13 -13
  24. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  30. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  31. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  32. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  33. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  35. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  37. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  38. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +2 -2
  39. metaflow-stubs/plugins/aws/__init__.pyi +3 -3
  40. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  47. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  48. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  56. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +2 -2
  57. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  58. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  60. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  61. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  62. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  63. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  64. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  66. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  71. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  73. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  77. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  78. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  79. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  80. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  81. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  82. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  83. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  84. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  85. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  86. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  87. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  88. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  89. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  90. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  91. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  93. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  94. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  95. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  96. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  97. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  98. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  99. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
  100. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  101. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  102. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  105. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  106. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  107. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  109. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/pypi/conda_environment.pyi +6 -6
  111. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  112. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  113. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  114. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  117. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  118. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  119. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  120. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  121. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  122. metaflow-stubs/pylint_wrapper.pyi +2 -2
  123. metaflow-stubs/runner/__init__.pyi +2 -2
  124. metaflow-stubs/runner/deployer.pyi +5 -5
  125. metaflow-stubs/runner/deployer_impl.pyi +2 -2
  126. metaflow-stubs/runner/metaflow_runner.pyi +3 -3
  127. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  128. metaflow-stubs/runner/nbrun.pyi +2 -2
  129. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  130. metaflow-stubs/runner/utils.pyi +3 -3
  131. metaflow-stubs/system/__init__.pyi +2 -2
  132. metaflow-stubs/system/system_logger.pyi +3 -3
  133. metaflow-stubs/system/system_monitor.pyi +2 -2
  134. metaflow-stubs/tagging_util.pyi +2 -2
  135. metaflow-stubs/tuple_util.pyi +2 -2
  136. metaflow-stubs/user_configs/__init__.pyi +2 -2
  137. metaflow-stubs/user_configs/config_decorators.pyi +5 -5
  138. metaflow-stubs/user_configs/config_options.pyi +2 -2
  139. metaflow-stubs/user_configs/config_parameters.pyi +5 -5
  140. metaflow-stubs/version.pyi +2 -2
  141. {metaflow_stubs-2.14.0.dist-info → metaflow_stubs-2.14.2.dist-info}/METADATA +2 -2
  142. metaflow_stubs-2.14.2.dist-info/RECORD +145 -0
  143. metaflow_stubs-2.14.0.dist-info/RECORD +0 -145
  144. {metaflow_stubs-2.14.0.dist-info → metaflow_stubs-2.14.2.dist-info}/WHEEL +0 -0
  145. {metaflow_stubs-2.14.0.dist-info → metaflow_stubs-2.14.2.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.14.0 #
4
- # Generated on 2025-02-11T19:31:25.349322 #
3
+ # MF version: 2.14.2 #
4
+ # Generated on 2025-02-21T20:07:35.058121 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -35,8 +35,8 @@ from .user_configs.config_parameters import ConfigValue as ConfigValue
35
35
  from .user_configs.config_parameters import config_expr as config_expr
36
36
  from .user_configs.config_decorators import CustomFlowDecorator as CustomFlowDecorator
37
37
  from .user_configs.config_decorators import CustomStepDecorator as CustomStepDecorator
38
- from . import events as events
39
38
  from . import tuple_util as tuple_util
39
+ from . import events as events
40
40
  from . import runner as runner
41
41
  from . import plugins as plugins
42
42
  from .plugins.datatools.s3.s3 import S3 as S3
@@ -142,6 +142,39 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
142
142
  """
143
143
  ...
144
144
 
145
+ @typing.overload
146
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
147
+ """
148
+ Specifies environment variables to be set prior to the execution of a step.
149
+
150
+
151
+ Parameters
152
+ ----------
153
+ vars : Dict[str, str], default {}
154
+ Dictionary of environment variables to set.
155
+ """
156
+ ...
157
+
158
+ @typing.overload
159
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
160
+ ...
161
+
162
+ @typing.overload
163
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
164
+ ...
165
+
166
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
167
+ """
168
+ Specifies environment variables to be set prior to the execution of a step.
169
+
170
+
171
+ Parameters
172
+ ----------
173
+ vars : Dict[str, str], default {}
174
+ Dictionary of environment variables to set.
175
+ """
176
+ ...
177
+
145
178
  @typing.overload
146
179
  def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
147
180
  """
@@ -197,62 +230,131 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
197
230
  """
198
231
  ...
199
232
 
233
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
234
+ """
235
+ Specifies that this step should execute on Kubernetes.
236
+
237
+
238
+ Parameters
239
+ ----------
240
+ cpu : int, default 1
241
+ Number of CPUs required for this step. If `@resources` is
242
+ also present, the maximum value from all decorators is used.
243
+ memory : int, default 4096
244
+ Memory size (in MB) required for this step. If
245
+ `@resources` is also present, the maximum value from all decorators is
246
+ used.
247
+ disk : int, default 10240
248
+ Disk size (in MB) required for this step. If
249
+ `@resources` is also present, the maximum value from all decorators is
250
+ used.
251
+ image : str, optional, default None
252
+ Docker image to use when launching on Kubernetes. If not specified, and
253
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
254
+ not, a default Docker image mapping to the current version of Python is used.
255
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
256
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
257
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
258
+ Kubernetes service account to use when launching pod in Kubernetes.
259
+ secrets : List[str], optional, default None
260
+ Kubernetes secrets to use when launching pod in Kubernetes. These
261
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
262
+ in Metaflow configuration.
263
+ node_selector: Union[Dict[str,str], str], optional, default None
264
+ Kubernetes node selector(s) to apply to the pod running the task.
265
+ Can be passed in as a comma separated string of values e.g.
266
+ 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
267
+ {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
268
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
269
+ Kubernetes namespace to use when launching pod in Kubernetes.
270
+ gpu : int, optional, default None
271
+ Number of GPUs required for this step. A value of zero implies that
272
+ the scheduled node should not have GPUs.
273
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
274
+ The vendor of the GPUs to be used for this step.
275
+ tolerations : List[str], default []
276
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
277
+ Kubernetes tolerations to use when launching pod in Kubernetes.
278
+ labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
279
+ Kubernetes labels to use when launching pod in Kubernetes.
280
+ annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
281
+ Kubernetes annotations to use when launching pod in Kubernetes.
282
+ use_tmpfs : bool, default False
283
+ This enables an explicit tmpfs mount for this step.
284
+ tmpfs_tempdir : bool, default True
285
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
286
+ tmpfs_size : int, optional, default: None
287
+ The value for the size (in MiB) of the tmpfs mount for this step.
288
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
289
+ memory allocated for this step.
290
+ tmpfs_path : str, optional, default /metaflow_temp
291
+ Path to tmpfs mount for this step.
292
+ persistent_volume_claims : Dict[str, str], optional, default None
293
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
294
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
295
+ shared_memory: int, optional
296
+ Shared memory size (in MiB) required for this step
297
+ port: int, optional
298
+ Port number to specify in the Kubernetes job object
299
+ compute_pool : str, optional, default None
300
+ Compute pool to be used for for this step.
301
+ If not specified, any accessible compute pool within the perimeter is used.
302
+ hostname_resolution_timeout: int, default 10 * 60
303
+ Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
304
+ Only applicable when @parallel is used.
305
+ qos: str, default: Burstable
306
+ Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
307
+ """
308
+ ...
309
+
200
310
  @typing.overload
201
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
311
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
202
312
  """
203
- Specifies the Conda environment for the step.
313
+ Specifies the PyPI packages for the step.
204
314
 
205
315
  Information in this decorator will augment any
206
- attributes set in the `@conda_base` flow-level decorator. Hence,
207
- you can use `@conda_base` to set packages required by all
208
- steps and use `@conda` to specify step-specific overrides.
316
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
317
+ you can use `@pypi_base` to set packages required by all
318
+ steps and use `@pypi` to specify step-specific overrides.
209
319
 
210
320
 
211
321
  Parameters
212
322
  ----------
213
- packages : Dict[str, str], default {}
323
+ packages : Dict[str, str], default: {}
214
324
  Packages to use for this step. The key is the name of the package
215
325
  and the value is the version to use.
216
- libraries : Dict[str, str], default {}
217
- Supported for backward compatibility. When used with packages, packages will take precedence.
218
- python : str, optional, default None
326
+ python : str, optional, default: None
219
327
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
220
328
  that the version used will correspond to the version of the Python interpreter used to start the run.
221
- disabled : bool, default False
222
- If set to True, disables @conda.
223
329
  """
224
330
  ...
225
331
 
226
332
  @typing.overload
227
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
333
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
228
334
  ...
229
335
 
230
336
  @typing.overload
231
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
337
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
232
338
  ...
233
339
 
234
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
340
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
235
341
  """
236
- Specifies the Conda environment for the step.
342
+ Specifies the PyPI packages for the step.
237
343
 
238
344
  Information in this decorator will augment any
239
- attributes set in the `@conda_base` flow-level decorator. Hence,
240
- you can use `@conda_base` to set packages required by all
241
- steps and use `@conda` to specify step-specific overrides.
345
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
346
+ you can use `@pypi_base` to set packages required by all
347
+ steps and use `@pypi` to specify step-specific overrides.
242
348
 
243
349
 
244
350
  Parameters
245
351
  ----------
246
- packages : Dict[str, str], default {}
352
+ packages : Dict[str, str], default: {}
247
353
  Packages to use for this step. The key is the name of the package
248
354
  and the value is the version to use.
249
- libraries : Dict[str, str], default {}
250
- Supported for backward compatibility. When used with packages, packages will take precedence.
251
- python : str, optional, default None
355
+ python : str, optional, default: None
252
356
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
253
357
  that the version used will correspond to the version of the Python interpreter used to start the run.
254
- disabled : bool, default False
255
- If set to True, disables @conda.
256
358
  """
257
359
  ...
258
360
 
@@ -276,124 +378,93 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
276
378
  ...
277
379
 
278
380
  @typing.overload
279
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
381
+ def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
280
382
  """
281
- Specifies the PyPI packages for the step.
383
+ Specifies the resources needed when executing this step.
282
384
 
283
- Information in this decorator will augment any
284
- attributes set in the `@pyi_base` flow-level decorator. Hence,
285
- you can use `@pypi_base` to set packages required by all
286
- steps and use `@pypi` to specify step-specific overrides.
385
+ Use `@resources` to specify the resource requirements
386
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
387
+
388
+ You can choose the compute layer on the command line by executing e.g.
389
+ ```
390
+ python myflow.py run --with batch
391
+ ```
392
+ or
393
+ ```
394
+ python myflow.py run --with kubernetes
395
+ ```
396
+ which executes the flow on the desired system using the
397
+ requirements specified in `@resources`.
287
398
 
288
399
 
289
400
  Parameters
290
401
  ----------
291
- packages : Dict[str, str], default: {}
292
- Packages to use for this step. The key is the name of the package
293
- and the value is the version to use.
294
- python : str, optional, default: None
295
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
296
- that the version used will correspond to the version of the Python interpreter used to start the run.
402
+ cpu : int, default 1
403
+ Number of CPUs required for this step.
404
+ gpu : int, optional, default None
405
+ Number of GPUs required for this step.
406
+ disk : int, optional, default None
407
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
408
+ memory : int, default 4096
409
+ Memory size (in MB) required for this step.
410
+ shared_memory : int, optional, default None
411
+ The value for the size (in MiB) of the /dev/shm volume for this step.
412
+ This parameter maps to the `--shm-size` option in Docker.
297
413
  """
298
414
  ...
299
415
 
300
416
  @typing.overload
301
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
417
+ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
302
418
  ...
303
419
 
304
420
  @typing.overload
305
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
421
+ def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
306
422
  ...
307
423
 
308
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
424
+ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
309
425
  """
310
- Specifies the PyPI packages for the step.
426
+ Specifies the resources needed when executing this step.
311
427
 
312
- Information in this decorator will augment any
313
- attributes set in the `@pyi_base` flow-level decorator. Hence,
314
- you can use `@pypi_base` to set packages required by all
315
- steps and use `@pypi` to specify step-specific overrides.
428
+ Use `@resources` to specify the resource requirements
429
+ independently of the specific compute layer (`@batch`, `@kubernetes`).
430
+
431
+ You can choose the compute layer on the command line by executing e.g.
432
+ ```
433
+ python myflow.py run --with batch
434
+ ```
435
+ or
436
+ ```
437
+ python myflow.py run --with kubernetes
438
+ ```
439
+ which executes the flow on the desired system using the
440
+ requirements specified in `@resources`.
316
441
 
317
442
 
318
443
  Parameters
319
444
  ----------
320
- packages : Dict[str, str], default: {}
321
- Packages to use for this step. The key is the name of the package
322
- and the value is the version to use.
323
- python : str, optional, default: None
324
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
325
- that the version used will correspond to the version of the Python interpreter used to start the run.
445
+ cpu : int, default 1
446
+ Number of CPUs required for this step.
447
+ gpu : int, optional, default None
448
+ Number of GPUs required for this step.
449
+ disk : int, optional, default None
450
+ Disk size (in MB) required for this step. Only applies on Kubernetes.
451
+ memory : int, default 4096
452
+ Memory size (in MB) required for this step.
453
+ shared_memory : int, optional, default None
454
+ The value for the size (in MiB) of the /dev/shm volume for this step.
455
+ This parameter maps to the `--shm-size` option in Docker.
326
456
  """
327
457
  ...
328
458
 
329
459
  @typing.overload
330
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
460
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
331
461
  """
332
- Specifies a timeout for your step.
333
-
334
- This decorator is useful if this step may hang indefinitely.
335
-
336
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
337
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
338
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
462
+ Specifies that the step will success under all circumstances.
339
463
 
340
- Note that all the values specified in parameters are added together so if you specify
341
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
342
-
343
-
344
- Parameters
345
- ----------
346
- seconds : int, default 0
347
- Number of seconds to wait prior to timing out.
348
- minutes : int, default 0
349
- Number of minutes to wait prior to timing out.
350
- hours : int, default 0
351
- Number of hours to wait prior to timing out.
352
- """
353
- ...
354
-
355
- @typing.overload
356
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
357
- ...
358
-
359
- @typing.overload
360
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
361
- ...
362
-
363
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
364
- """
365
- Specifies a timeout for your step.
366
-
367
- This decorator is useful if this step may hang indefinitely.
368
-
369
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
370
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
371
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
372
-
373
- Note that all the values specified in parameters are added together so if you specify
374
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
375
-
376
-
377
- Parameters
378
- ----------
379
- seconds : int, default 0
380
- Number of seconds to wait prior to timing out.
381
- minutes : int, default 0
382
- Number of minutes to wait prior to timing out.
383
- hours : int, default 0
384
- Number of hours to wait prior to timing out.
385
- """
386
- ...
387
-
388
- @typing.overload
389
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
390
- """
391
- Specifies that the step will success under all circumstances.
392
-
393
- The decorator will create an optional artifact, specified by `var`, which
394
- contains the exception raised. You can use it to detect the presence
395
- of errors, indicating that all happy-path artifacts produced by the step
396
- are missing.
464
+ The decorator will create an optional artifact, specified by `var`, which
465
+ contains the exception raised. You can use it to detect the presence
466
+ of errors, indicating that all happy-path artifacts produced by the step
467
+ are missing.
397
468
 
398
469
 
399
470
  Parameters
@@ -437,226 +508,61 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
437
508
  ...
438
509
 
439
510
  @typing.overload
440
- def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
441
- """
442
- Specifies the resources needed when executing this step.
443
-
444
- Use `@resources` to specify the resource requirements
445
- independently of the specific compute layer (`@batch`, `@kubernetes`).
446
-
447
- You can choose the compute layer on the command line by executing e.g.
448
- ```
449
- python myflow.py run --with batch
450
- ```
451
- or
452
- ```
453
- python myflow.py run --with kubernetes
454
- ```
455
- which executes the flow on the desired system using the
456
- requirements specified in `@resources`.
457
-
458
-
459
- Parameters
460
- ----------
461
- cpu : int, default 1
462
- Number of CPUs required for this step.
463
- gpu : int, optional, default None
464
- Number of GPUs required for this step.
465
- disk : int, optional, default None
466
- Disk size (in MB) required for this step. Only applies on Kubernetes.
467
- memory : int, default 4096
468
- Memory size (in MB) required for this step.
469
- shared_memory : int, optional, default None
470
- The value for the size (in MiB) of the /dev/shm volume for this step.
471
- This parameter maps to the `--shm-size` option in Docker.
472
- """
473
- ...
474
-
475
- @typing.overload
476
- def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
477
- ...
478
-
479
- @typing.overload
480
- def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
481
- ...
482
-
483
- def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
484
- """
485
- Specifies the resources needed when executing this step.
486
-
487
- Use `@resources` to specify the resource requirements
488
- independently of the specific compute layer (`@batch`, `@kubernetes`).
489
-
490
- You can choose the compute layer on the command line by executing e.g.
491
- ```
492
- python myflow.py run --with batch
493
- ```
494
- or
495
- ```
496
- python myflow.py run --with kubernetes
497
- ```
498
- which executes the flow on the desired system using the
499
- requirements specified in `@resources`.
500
-
501
-
502
- Parameters
503
- ----------
504
- cpu : int, default 1
505
- Number of CPUs required for this step.
506
- gpu : int, optional, default None
507
- Number of GPUs required for this step.
508
- disk : int, optional, default None
509
- Disk size (in MB) required for this step. Only applies on Kubernetes.
510
- memory : int, default 4096
511
- Memory size (in MB) required for this step.
512
- shared_memory : int, optional, default None
513
- The value for the size (in MiB) of the /dev/shm volume for this step.
514
- This parameter maps to the `--shm-size` option in Docker.
515
- """
516
- ...
517
-
518
- @typing.overload
519
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
520
- """
521
- Specifies environment variables to be set prior to the execution of a step.
522
-
523
-
524
- Parameters
525
- ----------
526
- vars : Dict[str, str], default {}
527
- Dictionary of environment variables to set.
528
- """
529
- ...
530
-
531
- @typing.overload
532
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
533
- ...
534
-
535
- @typing.overload
536
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
537
- ...
538
-
539
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
511
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
540
512
  """
541
- Specifies environment variables to be set prior to the execution of a step.
542
-
513
+ Specifies the Conda environment for the step.
543
514
 
544
- Parameters
545
- ----------
546
- vars : Dict[str, str], default {}
547
- Dictionary of environment variables to set.
548
- """
549
- ...
550
-
551
- @typing.overload
552
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
553
- """
554
- Specifies secrets to be retrieved and injected as environment variables prior to
555
- the execution of a step.
515
+ Information in this decorator will augment any
516
+ attributes set in the `@conda_base` flow-level decorator. Hence,
517
+ you can use `@conda_base` to set packages required by all
518
+ steps and use `@conda` to specify step-specific overrides.
556
519
 
557
520
 
558
521
  Parameters
559
522
  ----------
560
- sources : List[Union[str, Dict[str, Any]]], default: []
561
- List of secret specs, defining how the secrets are to be retrieved
523
+ packages : Dict[str, str], default {}
524
+ Packages to use for this step. The key is the name of the package
525
+ and the value is the version to use.
526
+ libraries : Dict[str, str], default {}
527
+ Supported for backward compatibility. When used with packages, packages will take precedence.
528
+ python : str, optional, default None
529
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
530
+ that the version used will correspond to the version of the Python interpreter used to start the run.
531
+ disabled : bool, default False
532
+ If set to True, disables @conda.
562
533
  """
563
534
  ...
564
535
 
565
536
  @typing.overload
566
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
537
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
567
538
  ...
568
539
 
569
540
  @typing.overload
570
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
541
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
571
542
  ...
572
543
 
573
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
544
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
574
545
  """
575
- Specifies secrets to be retrieved and injected as environment variables prior to
576
- the execution of a step.
577
-
546
+ Specifies the Conda environment for the step.
578
547
 
579
- Parameters
580
- ----------
581
- sources : List[Union[str, Dict[str, Any]]], default: []
582
- List of secret specs, defining how the secrets are to be retrieved
583
- """
584
- ...
585
-
586
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = 'KUBERNETES_IMAGE_PULL_POLICY', service_account: str = 'METAFLOW_KUBERNETES_SERVICE_ACCOUNT', secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = 'METAFLOW_KUBERNETES_NAMESPACE', gpu: typing.Optional[int] = None, gpu_vendor: str = 'KUBERNETES_GPU_VENDOR', tolerations: typing.List[str] = [], labels: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_LABELS', annotations: typing.Dict[str, str] = 'METAFLOW_KUBERNETES_ANNOTATIONS', use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = '/metaflow_temp', persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600, qos: str = 'Burstable') -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
587
- """
588
- Specifies that this step should execute on Kubernetes.
548
+ Information in this decorator will augment any
549
+ attributes set in the `@conda_base` flow-level decorator. Hence,
550
+ you can use `@conda_base` to set packages required by all
551
+ steps and use `@conda` to specify step-specific overrides.
589
552
 
590
553
 
591
554
  Parameters
592
555
  ----------
593
- cpu : int, default 1
594
- Number of CPUs required for this step. If `@resources` is
595
- also present, the maximum value from all decorators is used.
596
- memory : int, default 4096
597
- Memory size (in MB) required for this step. If
598
- `@resources` is also present, the maximum value from all decorators is
599
- used.
600
- disk : int, default 10240
601
- Disk size (in MB) required for this step. If
602
- `@resources` is also present, the maximum value from all decorators is
603
- used.
604
- image : str, optional, default None
605
- Docker image to use when launching on Kubernetes. If not specified, and
606
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
607
- not, a default Docker image mapping to the current version of Python is used.
608
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
609
- If given, the imagePullPolicy to be applied to the Docker image of the step.
610
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
611
- Kubernetes service account to use when launching pod in Kubernetes.
612
- secrets : List[str], optional, default None
613
- Kubernetes secrets to use when launching pod in Kubernetes. These
614
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
615
- in Metaflow configuration.
616
- node_selector: Union[Dict[str,str], str], optional, default None
617
- Kubernetes node selector(s) to apply to the pod running the task.
618
- Can be passed in as a comma separated string of values e.g.
619
- 'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
620
- {'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
621
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
622
- Kubernetes namespace to use when launching pod in Kubernetes.
623
- gpu : int, optional, default None
624
- Number of GPUs required for this step. A value of zero implies that
625
- the scheduled node should not have GPUs.
626
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
627
- The vendor of the GPUs to be used for this step.
628
- tolerations : List[str], default []
629
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
630
- Kubernetes tolerations to use when launching pod in Kubernetes.
631
- labels: Dict[str, str], default: METAFLOW_KUBERNETES_LABELS
632
- Kubernetes labels to use when launching pod in Kubernetes.
633
- annotations: Dict[str, str], default: METAFLOW_KUBERNETES_ANNOTATIONS
634
- Kubernetes annotations to use when launching pod in Kubernetes.
635
- use_tmpfs : bool, default False
636
- This enables an explicit tmpfs mount for this step.
637
- tmpfs_tempdir : bool, default True
638
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
639
- tmpfs_size : int, optional, default: None
640
- The value for the size (in MiB) of the tmpfs mount for this step.
641
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
642
- memory allocated for this step.
643
- tmpfs_path : str, optional, default /metaflow_temp
644
- Path to tmpfs mount for this step.
645
- persistent_volume_claims : Dict[str, str], optional, default None
646
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
647
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
648
- shared_memory: int, optional
649
- Shared memory size (in MiB) required for this step
650
- port: int, optional
651
- Port number to specify in the Kubernetes job object
652
- compute_pool : str, optional, default None
653
- Compute pool to be used for for this step.
654
- If not specified, any accessible compute pool within the perimeter is used.
655
- hostname_resolution_timeout: int, default 10 * 60
656
- Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
657
- Only applicable when @parallel is used.
658
- qos: str, default: Burstable
659
- Quality of Service class to assign to the pod. Supported values are: Guaranteed, Burstable, BestEffort
556
+ packages : Dict[str, str], default {}
557
+ Packages to use for this step. The key is the name of the package
558
+ and the value is the version to use.
559
+ libraries : Dict[str, str], default {}
560
+ Supported for backward compatibility. When used with packages, packages will take precedence.
561
+ python : str, optional, default None
562
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
563
+ that the version used will correspond to the version of the Python interpreter used to start the run.
564
+ disabled : bool, default False
565
+ If set to True, disables @conda.
660
566
  """
661
567
  ...
662
568
 
@@ -809,6 +715,100 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
809
715
  """
810
716
  ...
811
717
 
718
+ @typing.overload
719
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
720
+ """
721
+ Specifies secrets to be retrieved and injected as environment variables prior to
722
+ the execution of a step.
723
+
724
+
725
+ Parameters
726
+ ----------
727
+ sources : List[Union[str, Dict[str, Any]]], default: []
728
+ List of secret specs, defining how the secrets are to be retrieved
729
+ """
730
+ ...
731
+
732
+ @typing.overload
733
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
734
+ ...
735
+
736
+ @typing.overload
737
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
738
+ ...
739
+
740
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
741
+ """
742
+ Specifies secrets to be retrieved and injected as environment variables prior to
743
+ the execution of a step.
744
+
745
+
746
+ Parameters
747
+ ----------
748
+ sources : List[Union[str, Dict[str, Any]]], default: []
749
+ List of secret specs, defining how the secrets are to be retrieved
750
+ """
751
+ ...
752
+
753
+ @typing.overload
754
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
755
+ """
756
+ Specifies a timeout for your step.
757
+
758
+ This decorator is useful if this step may hang indefinitely.
759
+
760
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
761
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
762
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
763
+
764
+ Note that all the values specified in parameters are added together so if you specify
765
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
766
+
767
+
768
+ Parameters
769
+ ----------
770
+ seconds : int, default 0
771
+ Number of seconds to wait prior to timing out.
772
+ minutes : int, default 0
773
+ Number of minutes to wait prior to timing out.
774
+ hours : int, default 0
775
+ Number of hours to wait prior to timing out.
776
+ """
777
+ ...
778
+
779
+ @typing.overload
780
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
781
+ ...
782
+
783
+ @typing.overload
784
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
785
+ ...
786
+
787
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
788
+ """
789
+ Specifies a timeout for your step.
790
+
791
+ This decorator is useful if this step may hang indefinitely.
792
+
793
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
794
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
795
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
796
+
797
+ Note that all the values specified in parameters are added together so if you specify
798
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
799
+
800
+
801
+ Parameters
802
+ ----------
803
+ seconds : int, default 0
804
+ Number of seconds to wait prior to timing out.
805
+ minutes : int, default 0
806
+ Number of minutes to wait prior to timing out.
807
+ hours : int, default 0
808
+ Number of hours to wait prior to timing out.
809
+ """
810
+ ...
811
+
812
812
  @typing.overload
813
813
  def card(*, type: str = 'default', id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
814
814
  """
@@ -858,44 +858,89 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
858
858
  """
859
859
  ...
860
860
 
861
+ def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
862
+ """
863
+ Specifies what flows belong to the same project.
864
+
865
+ A project-specific namespace is created for all flows that
866
+ use the same `@project(name)`.
867
+
868
+
869
+ Parameters
870
+ ----------
871
+ name : str
872
+ Project name. Make sure that the name is unique amongst all
873
+ projects that use the same production scheduler. The name may
874
+ contain only lowercase alphanumeric characters and underscores.
875
+
876
+ branch : Optional[str], default None
877
+ The branch to use. If not specified, the branch is set to
878
+ `user.<username>` unless `production` is set to `True`. This can
879
+ also be set on the command line using `--branch` as a top-level option.
880
+ It is an error to specify `branch` in the decorator and on the command line.
881
+
882
+ production : bool, default False
883
+ Whether or not the branch is the production branch. This can also be set on the
884
+ command line using `--production` as a top-level option. It is an error to specify
885
+ `production` in the decorator and on the command line.
886
+ The project branch name will be:
887
+ - if `branch` is specified:
888
+ - if `production` is True: `prod.<branch>`
889
+ - if `production` is False: `test.<branch>`
890
+ - if `branch` is not specified:
891
+ - if `production` is True: `prod`
892
+ - if `production` is False: `user.<username>`
893
+ """
894
+ ...
895
+
861
896
  @typing.overload
862
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
897
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
863
898
  """
864
- Specifies the PyPI packages for all steps of the flow.
899
+ Specifies the Conda environment for all steps of the flow.
900
+
901
+ Use `@conda_base` to set common libraries required by all
902
+ steps and use `@conda` to specify step-specific additions.
865
903
 
866
- Use `@pypi_base` to set common packages required by all
867
- steps and use `@pypi` to specify step-specific overrides.
868
904
 
869
905
  Parameters
870
906
  ----------
871
- packages : Dict[str, str], default: {}
907
+ packages : Dict[str, str], default {}
872
908
  Packages to use for this flow. The key is the name of the package
873
909
  and the value is the version to use.
874
- python : str, optional, default: None
910
+ libraries : Dict[str, str], default {}
911
+ Supported for backward compatibility. When used with packages, packages will take precedence.
912
+ python : str, optional, default None
875
913
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
876
914
  that the version used will correspond to the version of the Python interpreter used to start the run.
915
+ disabled : bool, default False
916
+ If set to True, disables Conda.
877
917
  """
878
918
  ...
879
919
 
880
920
  @typing.overload
881
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
921
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
882
922
  ...
883
923
 
884
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
924
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
885
925
  """
886
- Specifies the PyPI packages for all steps of the flow.
926
+ Specifies the Conda environment for all steps of the flow.
927
+
928
+ Use `@conda_base` to set common libraries required by all
929
+ steps and use `@conda` to specify step-specific additions.
887
930
 
888
- Use `@pypi_base` to set common packages required by all
889
- steps and use `@pypi` to specify step-specific overrides.
890
931
 
891
932
  Parameters
892
933
  ----------
893
- packages : Dict[str, str], default: {}
934
+ packages : Dict[str, str], default {}
894
935
  Packages to use for this flow. The key is the name of the package
895
936
  and the value is the version to use.
896
- python : str, optional, default: None
937
+ libraries : Dict[str, str], default {}
938
+ Supported for backward compatibility. When used with packages, packages will take precedence.
939
+ python : str, optional, default None
897
940
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
898
941
  that the version used will correspond to the version of the Python interpreter used to start the run.
942
+ disabled : bool, default False
943
+ If set to True, disables Conda.
899
944
  """
900
945
  ...
901
946
 
@@ -950,38 +995,44 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
950
995
  """
951
996
  ...
952
997
 
953
- def project(*, name: str, branch: typing.Optional[str] = None, production: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
998
+ @typing.overload
999
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
954
1000
  """
955
- Specifies what flows belong to the same project.
1001
+ Specifies the PyPI packages for all steps of the flow.
956
1002
 
957
- A project-specific namespace is created for all flows that
958
- use the same `@project(name)`.
1003
+ Use `@pypi_base` to set common packages required by all
1004
+ steps and use `@pypi` to specify step-specific overrides.
1005
+
1006
+ Parameters
1007
+ ----------
1008
+ packages : Dict[str, str], default: {}
1009
+ Packages to use for this flow. The key is the name of the package
1010
+ and the value is the version to use.
1011
+ python : str, optional, default: None
1012
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1013
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1014
+ """
1015
+ ...
1016
+
1017
+ @typing.overload
1018
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1019
+ ...
1020
+
1021
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1022
+ """
1023
+ Specifies the PyPI packages for all steps of the flow.
959
1024
 
1025
+ Use `@pypi_base` to set common packages required by all
1026
+ steps and use `@pypi` to specify step-specific overrides.
960
1027
 
961
1028
  Parameters
962
1029
  ----------
963
- name : str
964
- Project name. Make sure that the name is unique amongst all
965
- projects that use the same production scheduler. The name may
966
- contain only lowercase alphanumeric characters and underscores.
967
-
968
- branch : Optional[str], default None
969
- The branch to use. If not specified, the branch is set to
970
- `user.<username>` unless `production` is set to `True`. This can
971
- also be set on the command line using `--branch` as a top-level option.
972
- It is an error to specify `branch` in the decorator and on the command line.
973
-
974
- production : bool, default False
975
- Whether or not the branch is the production branch. This can also be set on the
976
- command line using `--production` as a top-level option. It is an error to specify
977
- `production` in the decorator and on the command line.
978
- The project branch name will be:
979
- - if `branch` is specified:
980
- - if `production` is True: `prod.<branch>`
981
- - if `production` is False: `test.<branch>`
982
- - if `branch` is not specified:
983
- - if `production` is True: `prod`
984
- - if `production` is False: `user.<username>`
1030
+ packages : Dict[str, str], default: {}
1031
+ Packages to use for this flow. The key is the name of the package
1032
+ and the value is the version to use.
1033
+ python : str, optional, default: None
1034
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1035
+ that the version used will correspond to the version of the Python interpreter used to start the run.
985
1036
  """
986
1037
  ...
987
1038
 
@@ -1029,157 +1080,98 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1029
1080
  ...
1030
1081
 
1031
1082
  @typing.overload
1032
- def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1083
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1033
1084
  """
1034
- Specifies the flow(s) that this flow depends on.
1085
+ Specifies the event(s) that this flow depends on.
1035
1086
 
1036
1087
  ```
1037
- @trigger_on_finish(flow='FooFlow')
1088
+ @trigger(event='foo')
1038
1089
  ```
1039
1090
  or
1040
1091
  ```
1041
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1092
+ @trigger(events=['foo', 'bar'])
1042
1093
  ```
1043
- This decorator respects the @project decorator and triggers the flow
1044
- when upstream runs within the same namespace complete successfully
1045
1094
 
1046
- Additionally, you can specify project aware upstream flow dependencies
1047
- by specifying the fully qualified project_flow_name.
1095
+ Additionally, you can specify the parameter mappings
1096
+ to map event payload to Metaflow parameters for the flow.
1048
1097
  ```
1049
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1098
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1050
1099
  ```
1051
1100
  or
1052
1101
  ```
1053
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1102
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1103
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1054
1104
  ```
1055
1105
 
1056
- You can also specify just the project or project branch (other values will be
1057
- inferred from the current project or project branch):
1106
+ 'parameters' can also be a list of strings and tuples like so:
1058
1107
  ```
1059
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1108
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1109
+ ```
1110
+ This is equivalent to:
1111
+ ```
1112
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1060
1113
  ```
1061
-
1062
- Note that `branch` is typically one of:
1063
- - `prod`
1064
- - `user.bob`
1065
- - `test.my_experiment`
1066
- - `prod.staging`
1067
1114
 
1068
1115
 
1069
1116
  Parameters
1070
1117
  ----------
1071
- flow : Union[str, Dict[str, str]], optional, default None
1072
- Upstream flow dependency for this flow.
1073
- flows : List[Union[str, Dict[str, str]]], default []
1074
- Upstream flow dependencies for this flow.
1118
+ event : Union[str, Dict[str, Any]], optional, default None
1119
+ Event dependency for this flow.
1120
+ events : List[Union[str, Dict[str, Any]]], default []
1121
+ Events dependency for this flow.
1075
1122
  options : Dict[str, Any], default {}
1076
1123
  Backend-specific configuration for tuning eventing behavior.
1077
1124
  """
1078
1125
  ...
1079
1126
 
1080
1127
  @typing.overload
1081
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1128
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1082
1129
  ...
1083
1130
 
1084
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1131
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1085
1132
  """
1086
- Specifies the flow(s) that this flow depends on.
1133
+ Specifies the event(s) that this flow depends on.
1087
1134
 
1088
1135
  ```
1089
- @trigger_on_finish(flow='FooFlow')
1136
+ @trigger(event='foo')
1090
1137
  ```
1091
1138
  or
1092
1139
  ```
1093
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1140
+ @trigger(events=['foo', 'bar'])
1094
1141
  ```
1095
- This decorator respects the @project decorator and triggers the flow
1096
- when upstream runs within the same namespace complete successfully
1097
1142
 
1098
- Additionally, you can specify project aware upstream flow dependencies
1099
- by specifying the fully qualified project_flow_name.
1143
+ Additionally, you can specify the parameter mappings
1144
+ to map event payload to Metaflow parameters for the flow.
1100
1145
  ```
1101
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1146
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1102
1147
  ```
1103
1148
  or
1104
1149
  ```
1105
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1150
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1151
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1106
1152
  ```
1107
1153
 
1108
- You can also specify just the project or project branch (other values will be
1109
- inferred from the current project or project branch):
1154
+ 'parameters' can also be a list of strings and tuples like so:
1110
1155
  ```
1111
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1156
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1157
+ ```
1158
+ This is equivalent to:
1159
+ ```
1160
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1112
1161
  ```
1113
-
1114
- Note that `branch` is typically one of:
1115
- - `prod`
1116
- - `user.bob`
1117
- - `test.my_experiment`
1118
- - `prod.staging`
1119
1162
 
1120
1163
 
1121
1164
  Parameters
1122
1165
  ----------
1123
- flow : Union[str, Dict[str, str]], optional, default None
1124
- Upstream flow dependency for this flow.
1125
- flows : List[Union[str, Dict[str, str]]], default []
1126
- Upstream flow dependencies for this flow.
1166
+ event : Union[str, Dict[str, Any]], optional, default None
1167
+ Event dependency for this flow.
1168
+ events : List[Union[str, Dict[str, Any]]], default []
1169
+ Events dependency for this flow.
1127
1170
  options : Dict[str, Any], default {}
1128
1171
  Backend-specific configuration for tuning eventing behavior.
1129
1172
  """
1130
1173
  ...
1131
1174
 
1132
- @typing.overload
1133
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1134
- """
1135
- Specifies the Conda environment for all steps of the flow.
1136
-
1137
- Use `@conda_base` to set common libraries required by all
1138
- steps and use `@conda` to specify step-specific additions.
1139
-
1140
-
1141
- Parameters
1142
- ----------
1143
- packages : Dict[str, str], default {}
1144
- Packages to use for this flow. The key is the name of the package
1145
- and the value is the version to use.
1146
- libraries : Dict[str, str], default {}
1147
- Supported for backward compatibility. When used with packages, packages will take precedence.
1148
- python : str, optional, default None
1149
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1150
- that the version used will correspond to the version of the Python interpreter used to start the run.
1151
- disabled : bool, default False
1152
- If set to True, disables Conda.
1153
- """
1154
- ...
1155
-
1156
- @typing.overload
1157
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1158
- ...
1159
-
1160
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1161
- """
1162
- Specifies the Conda environment for all steps of the flow.
1163
-
1164
- Use `@conda_base` to set common libraries required by all
1165
- steps and use `@conda` to specify step-specific additions.
1166
-
1167
-
1168
- Parameters
1169
- ----------
1170
- packages : Dict[str, str], default {}
1171
- Packages to use for this flow. The key is the name of the package
1172
- and the value is the version to use.
1173
- libraries : Dict[str, str], default {}
1174
- Supported for backward compatibility. When used with packages, packages will take precedence.
1175
- python : str, optional, default None
1176
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1177
- that the version used will correspond to the version of the Python interpreter used to start the run.
1178
- disabled : bool, default False
1179
- If set to True, disables Conda.
1180
- """
1181
- ...
1182
-
1183
1175
  def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1184
1176
  """
1185
1177
  The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
@@ -1224,93 +1216,101 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1224
1216
  ...
1225
1217
 
1226
1218
  @typing.overload
1227
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1219
+ def trigger_on_finish(*, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1228
1220
  """
1229
- Specifies the event(s) that this flow depends on.
1221
+ Specifies the flow(s) that this flow depends on.
1230
1222
 
1231
1223
  ```
1232
- @trigger(event='foo')
1224
+ @trigger_on_finish(flow='FooFlow')
1233
1225
  ```
1234
1226
  or
1235
1227
  ```
1236
- @trigger(events=['foo', 'bar'])
1228
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1237
1229
  ```
1230
+ This decorator respects the @project decorator and triggers the flow
1231
+ when upstream runs within the same namespace complete successfully
1238
1232
 
1239
- Additionally, you can specify the parameter mappings
1240
- to map event payload to Metaflow parameters for the flow.
1233
+ Additionally, you can specify project aware upstream flow dependencies
1234
+ by specifying the fully qualified project_flow_name.
1241
1235
  ```
1242
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1236
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1243
1237
  ```
1244
1238
  or
1245
1239
  ```
1246
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1247
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1240
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1248
1241
  ```
1249
1242
 
1250
- 'parameters' can also be a list of strings and tuples like so:
1251
- ```
1252
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1253
- ```
1254
- This is equivalent to:
1243
+ You can also specify just the project or project branch (other values will be
1244
+ inferred from the current project or project branch):
1255
1245
  ```
1256
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1246
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1257
1247
  ```
1258
1248
 
1249
+ Note that `branch` is typically one of:
1250
+ - `prod`
1251
+ - `user.bob`
1252
+ - `test.my_experiment`
1253
+ - `prod.staging`
1254
+
1259
1255
 
1260
1256
  Parameters
1261
1257
  ----------
1262
- event : Union[str, Dict[str, Any]], optional, default None
1263
- Event dependency for this flow.
1264
- events : List[Union[str, Dict[str, Any]]], default []
1265
- Events dependency for this flow.
1258
+ flow : Union[str, Dict[str, str]], optional, default None
1259
+ Upstream flow dependency for this flow.
1260
+ flows : List[Union[str, Dict[str, str]]], default []
1261
+ Upstream flow dependencies for this flow.
1266
1262
  options : Dict[str, Any], default {}
1267
1263
  Backend-specific configuration for tuning eventing behavior.
1268
1264
  """
1269
1265
  ...
1270
1266
 
1271
1267
  @typing.overload
1272
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1268
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1273
1269
  ...
1274
1270
 
1275
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1271
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[typing.Dict[str, str], str, None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1276
1272
  """
1277
- Specifies the event(s) that this flow depends on.
1273
+ Specifies the flow(s) that this flow depends on.
1278
1274
 
1279
1275
  ```
1280
- @trigger(event='foo')
1276
+ @trigger_on_finish(flow='FooFlow')
1281
1277
  ```
1282
1278
  or
1283
1279
  ```
1284
- @trigger(events=['foo', 'bar'])
1280
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1285
1281
  ```
1282
+ This decorator respects the @project decorator and triggers the flow
1283
+ when upstream runs within the same namespace complete successfully
1286
1284
 
1287
- Additionally, you can specify the parameter mappings
1288
- to map event payload to Metaflow parameters for the flow.
1285
+ Additionally, you can specify project aware upstream flow dependencies
1286
+ by specifying the fully qualified project_flow_name.
1289
1287
  ```
1290
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1288
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1291
1289
  ```
1292
1290
  or
1293
1291
  ```
1294
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1295
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1292
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1296
1293
  ```
1297
1294
 
1298
- 'parameters' can also be a list of strings and tuples like so:
1299
- ```
1300
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1301
- ```
1302
- This is equivalent to:
1295
+ You can also specify just the project or project branch (other values will be
1296
+ inferred from the current project or project branch):
1303
1297
  ```
1304
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1298
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1305
1299
  ```
1306
1300
 
1301
+ Note that `branch` is typically one of:
1302
+ - `prod`
1303
+ - `user.bob`
1304
+ - `test.my_experiment`
1305
+ - `prod.staging`
1306
+
1307
1307
 
1308
1308
  Parameters
1309
1309
  ----------
1310
- event : Union[str, Dict[str, Any]], optional, default None
1311
- Event dependency for this flow.
1312
- events : List[Union[str, Dict[str, Any]]], default []
1313
- Events dependency for this flow.
1310
+ flow : Union[str, Dict[str, str]], optional, default None
1311
+ Upstream flow dependency for this flow.
1312
+ flows : List[Union[str, Dict[str, str]]], default []
1313
+ Upstream flow dependencies for this flow.
1314
1314
  options : Dict[str, Any], default {}
1315
1315
  Backend-specific configuration for tuning eventing behavior.
1316
1316
  """