ob-metaflow-stubs 5.8.1__py2.py3-none-any.whl → 5.8.2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. metaflow-stubs/__init__.pyi +462 -462
  2. metaflow-stubs/cards.pyi +4 -4
  3. metaflow-stubs/cli.pyi +1 -1
  4. metaflow-stubs/client/__init__.pyi +2 -2
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +1 -1
  7. metaflow-stubs/clone_util.pyi +1 -1
  8. metaflow-stubs/events.pyi +1 -1
  9. metaflow-stubs/exception.pyi +1 -1
  10. metaflow-stubs/flowspec.pyi +4 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +3 -3
  13. metaflow-stubs/metadata/metadata.pyi +1 -1
  14. metaflow-stubs/metadata/util.pyi +1 -1
  15. metaflow-stubs/metaflow_config.pyi +1 -1
  16. metaflow-stubs/metaflow_current.pyi +5 -5
  17. metaflow-stubs/mflog/mflog.pyi +1 -1
  18. metaflow-stubs/multicore_utils.pyi +1 -1
  19. metaflow-stubs/parameters.pyi +3 -3
  20. metaflow-stubs/plugins/__init__.pyi +2 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +1 -1
  22. metaflow-stubs/plugins/airflow/airflow_utils.pyi +1 -1
  23. metaflow-stubs/plugins/airflow/exception.pyi +1 -1
  24. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +1 -1
  25. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +1 -1
  26. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  28. metaflow-stubs/plugins/argo/__init__.pyi +1 -1
  29. metaflow-stubs/plugins/argo/argo_client.pyi +1 -1
  30. metaflow-stubs/plugins/argo/argo_events.pyi +1 -1
  31. metaflow-stubs/plugins/argo/argo_workflows.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +4 -4
  33. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +6 -6
  34. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  35. metaflow-stubs/plugins/aws/__init__.pyi +1 -1
  36. metaflow-stubs/plugins/aws/aws_client.pyi +1 -1
  37. metaflow-stubs/plugins/aws/aws_utils.pyi +1 -1
  38. metaflow-stubs/plugins/aws/batch/__init__.pyi +1 -1
  39. metaflow-stubs/plugins/aws/batch/batch.pyi +1 -1
  40. metaflow-stubs/plugins/aws/batch/batch_client.pyi +1 -1
  41. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +1 -1
  42. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +1 -1
  43. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  44. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +1 -1
  45. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +1 -1
  46. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +1 -1
  47. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +1 -1
  48. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +1 -1
  49. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +1 -1
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +1 -1
  51. metaflow-stubs/plugins/azure/__init__.pyi +1 -1
  52. metaflow-stubs/plugins/azure/azure_credential.pyi +1 -1
  53. metaflow-stubs/plugins/azure/azure_exceptions.pyi +1 -1
  54. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_utils.pyi +1 -1
  56. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +1 -1
  57. metaflow-stubs/plugins/azure/includefile_support.pyi +1 -1
  58. metaflow-stubs/plugins/cards/__init__.pyi +1 -1
  59. metaflow-stubs/plugins/cards/card_cli.pyi +3 -3
  60. metaflow-stubs/plugins/cards/card_client.pyi +1 -1
  61. metaflow-stubs/plugins/cards/card_creator.pyi +1 -1
  62. metaflow-stubs/plugins/cards/card_datastore.pyi +1 -1
  63. metaflow-stubs/plugins/cards/card_decorator.pyi +1 -1
  64. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +1 -1
  65. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_modules/card.pyi +1 -1
  67. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  68. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +1 -1
  69. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +1 -1
  70. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_resolver.pyi +1 -1
  72. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  73. metaflow-stubs/plugins/cards/exception.pyi +1 -1
  74. metaflow-stubs/plugins/catch_decorator.pyi +1 -1
  75. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  76. metaflow-stubs/plugins/datatools/local.pyi +1 -1
  77. metaflow-stubs/plugins/datatools/s3/__init__.pyi +1 -1
  78. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  79. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +1 -1
  80. metaflow-stubs/plugins/datatools/s3/s3util.pyi +1 -1
  81. metaflow-stubs/plugins/debug_logger.pyi +1 -1
  82. metaflow-stubs/plugins/debug_monitor.pyi +1 -1
  83. metaflow-stubs/plugins/environment_decorator.pyi +1 -1
  84. metaflow-stubs/plugins/events_decorator.pyi +1 -1
  85. metaflow-stubs/plugins/frameworks/__init__.pyi +1 -1
  86. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  87. metaflow-stubs/plugins/gcp/__init__.pyi +1 -1
  88. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +1 -1
  90. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +1 -1
  91. metaflow-stubs/plugins/gcp/gs_utils.pyi +1 -1
  92. metaflow-stubs/plugins/gcp/includefile_support.pyi +1 -1
  93. metaflow-stubs/plugins/kubernetes/__init__.pyi +1 -1
  94. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +1 -1
  95. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +1 -1
  97. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +1 -1
  98. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +1 -1
  99. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  100. metaflow-stubs/plugins/package_cli.pyi +1 -1
  101. metaflow-stubs/plugins/parallel_decorator.pyi +1 -1
  102. metaflow-stubs/plugins/perimeters.pyi +1 -1
  103. metaflow-stubs/plugins/project_decorator.pyi +1 -1
  104. metaflow-stubs/plugins/pypi/__init__.pyi +1 -1
  105. metaflow-stubs/plugins/pypi/conda_decorator.pyi +1 -1
  106. metaflow-stubs/plugins/pypi/conda_environment.pyi +1 -1
  107. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +1 -1
  108. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  109. metaflow-stubs/plugins/pypi/utils.pyi +1 -1
  110. metaflow-stubs/plugins/resources_decorator.pyi +1 -1
  111. metaflow-stubs/plugins/retry_decorator.pyi +1 -1
  112. metaflow-stubs/plugins/secrets/__init__.pyi +1 -1
  113. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +1 -1
  114. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +1 -1
  115. metaflow-stubs/plugins/storage_executor.pyi +1 -1
  116. metaflow-stubs/plugins/tag_cli.pyi +3 -3
  117. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/timeout_decorator.pyi +1 -1
  119. metaflow-stubs/procpoll.pyi +1 -1
  120. metaflow-stubs/profilers/__init__.pyi +1 -1
  121. metaflow-stubs/pylint_wrapper.pyi +1 -1
  122. metaflow-stubs/runner/__init__.pyi +1 -1
  123. metaflow-stubs/runner/deployer.pyi +1 -1
  124. metaflow-stubs/runner/metaflow_runner.pyi +2 -2
  125. metaflow-stubs/runner/nbdeploy.pyi +1 -1
  126. metaflow-stubs/runner/nbrun.pyi +1 -1
  127. metaflow-stubs/runner/subprocess_manager.pyi +1 -1
  128. metaflow-stubs/runner/utils.pyi +1 -1
  129. metaflow-stubs/system/__init__.pyi +1 -1
  130. metaflow-stubs/system/system_logger.pyi +1 -1
  131. metaflow-stubs/system/system_monitor.pyi +1 -1
  132. metaflow-stubs/tagging_util.pyi +1 -1
  133. metaflow-stubs/tuple_util.pyi +1 -1
  134. {ob_metaflow_stubs-5.8.1.dist-info → ob_metaflow_stubs-5.8.2.dist-info}/METADATA +1 -1
  135. ob_metaflow_stubs-5.8.2.dist-info/RECORD +138 -0
  136. ob_metaflow_stubs-5.8.1.dist-info/RECORD +0 -138
  137. {ob_metaflow_stubs-5.8.1.dist-info → ob_metaflow_stubs-5.8.2.dist-info}/WHEEL +0 -0
  138. {ob_metaflow_stubs-5.8.1.dist-info → ob_metaflow_stubs-5.8.2.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
3
  # MF version: 2.12.19.1+ob(v1) #
4
- # Generated on 2024-09-06T08:14:24.914849 #
4
+ # Generated on 2024-09-09T18:18:13.623870 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,15 +9,15 @@ from __future__ import annotations
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
11
  import metaflow.parameters
12
- import typing
13
12
  import datetime
14
- import metaflow.runner.metaflow_runner
13
+ import metaflow._vendor.click.types
14
+ import metaflow.metaflow_current
15
15
  import metaflow.datastore.inputs
16
16
  import metaflow.flowspec
17
+ import typing
18
+ import metaflow.runner.metaflow_runner
17
19
  import metaflow.events
18
20
  import metaflow.client.core
19
- import metaflow._vendor.click.types
20
- import metaflow.metaflow_current
21
21
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
22
22
  StepFlag = typing.NewType("StepFlag", bool)
23
23
 
@@ -437,133 +437,55 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
437
437
  """
438
438
  ...
439
439
 
440
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
441
- """
442
- Specifies that this step should execute on Kubernetes.
443
-
444
- Parameters
445
- ----------
446
- cpu : int, default 1
447
- Number of CPUs required for this step. If `@resources` is
448
- also present, the maximum value from all decorators is used.
449
- memory : int, default 4096
450
- Memory size (in MB) required for this step. If
451
- `@resources` is also present, the maximum value from all decorators is
452
- used.
453
- disk : int, default 10240
454
- Disk size (in MB) required for this step. If
455
- `@resources` is also present, the maximum value from all decorators is
456
- used.
457
- image : str, optional, default None
458
- Docker image to use when launching on Kubernetes. If not specified, and
459
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
460
- not, a default Docker image mapping to the current version of Python is used.
461
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
462
- If given, the imagePullPolicy to be applied to the Docker image of the step.
463
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
464
- Kubernetes service account to use when launching pod in Kubernetes.
465
- secrets : List[str], optional, default None
466
- Kubernetes secrets to use when launching pod in Kubernetes. These
467
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
468
- in Metaflow configuration.
469
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
470
- Kubernetes namespace to use when launching pod in Kubernetes.
471
- gpu : int, optional, default None
472
- Number of GPUs required for this step. A value of zero implies that
473
- the scheduled node should not have GPUs.
474
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
475
- The vendor of the GPUs to be used for this step.
476
- tolerations : List[str], default []
477
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
478
- Kubernetes tolerations to use when launching pod in Kubernetes.
479
- use_tmpfs : bool, default False
480
- This enables an explicit tmpfs mount for this step.
481
- tmpfs_tempdir : bool, default True
482
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
483
- tmpfs_size : int, optional, default: None
484
- The value for the size (in MiB) of the tmpfs mount for this step.
485
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
486
- memory allocated for this step.
487
- tmpfs_path : str, optional, default /metaflow_temp
488
- Path to tmpfs mount for this step.
489
- persistent_volume_claims : Dict[str, str], optional, default None
490
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
491
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
492
- shared_memory: int, optional
493
- Shared memory size (in MiB) required for this step
494
- port: int, optional
495
- Port number to specify in the Kubernetes job object
496
- compute_pool : str, optional, default None
497
- Compute pool to be used for for this step.
498
- If not specified, any accessible compute pool within the perimeter is used.
499
- """
500
- ...
501
-
502
440
  @typing.overload
503
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
441
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
504
442
  """
505
- Specifies the PyPI packages for the step.
506
-
507
- Information in this decorator will augment any
508
- attributes set in the `@pyi_base` flow-level decorator. Hence,
509
- you can use `@pypi_base` to set packages required by all
510
- steps and use `@pypi` to specify step-specific overrides.
511
-
512
- Parameters
513
- ----------
514
- packages : Dict[str, str], default: {}
515
- Packages to use for this step. The key is the name of the package
516
- and the value is the version to use.
517
- python : str, optional, default: None
518
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
519
- that the version used will correspond to the version of the Python interpreter used to start the run.
443
+ Decorator prototype for all step decorators. This function gets specialized
444
+ and imported for all decorators types by _import_plugin_decorators().
520
445
  """
521
446
  ...
522
447
 
523
448
  @typing.overload
524
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
449
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
525
450
  ...
526
451
 
527
- @typing.overload
528
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
452
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
453
+ """
454
+ Decorator prototype for all step decorators. This function gets specialized
455
+ and imported for all decorators types by _import_plugin_decorators().
456
+ """
529
457
  ...
530
458
 
531
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
459
+ @typing.overload
460
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
532
461
  """
533
- Specifies the PyPI packages for the step.
534
-
535
- Information in this decorator will augment any
536
- attributes set in the `@pyi_base` flow-level decorator. Hence,
537
- you can use `@pypi_base` to set packages required by all
538
- steps and use `@pypi` to specify step-specific overrides.
462
+ Specifies secrets to be retrieved and injected as environment variables prior to
463
+ the execution of a step.
539
464
 
540
465
  Parameters
541
466
  ----------
542
- packages : Dict[str, str], default: {}
543
- Packages to use for this step. The key is the name of the package
544
- and the value is the version to use.
545
- python : str, optional, default: None
546
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
547
- that the version used will correspond to the version of the Python interpreter used to start the run.
467
+ sources : List[Union[str, Dict[str, Any]]], default: []
468
+ List of secret specs, defining how the secrets are to be retrieved
548
469
  """
549
470
  ...
550
471
 
551
472
  @typing.overload
552
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
553
- """
554
- Decorator prototype for all step decorators. This function gets specialized
555
- and imported for all decorators types by _import_plugin_decorators().
556
- """
473
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
557
474
  ...
558
475
 
559
476
  @typing.overload
560
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
477
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
561
478
  ...
562
479
 
563
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
480
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
564
481
  """
565
- Decorator prototype for all step decorators. This function gets specialized
566
- and imported for all decorators types by _import_plugin_decorators().
482
+ Specifies secrets to be retrieved and injected as environment variables prior to
483
+ the execution of a step.
484
+
485
+ Parameters
486
+ ----------
487
+ sources : List[Union[str, Dict[str, Any]]], default: []
488
+ List of secret specs, defining how the secrets are to be retrieved
567
489
  """
568
490
  ...
569
491
 
@@ -619,88 +541,51 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
619
541
  ...
620
542
 
621
543
  @typing.overload
622
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
544
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
623
545
  """
624
- Specifies the number of times the task corresponding
625
- to a step needs to be retried.
626
-
627
- This decorator is useful for handling transient errors, such as networking issues.
628
- If your task contains operations that can't be retried safely, e.g. database updates,
629
- it is advisable to annotate it with `@retry(times=0)`.
546
+ Specifies the PyPI packages for the step.
630
547
 
631
- This can be used in conjunction with the `@catch` decorator. The `@catch`
632
- decorator will execute a no-op task after all retries have been exhausted,
633
- ensuring that the flow execution can continue.
548
+ Information in this decorator will augment any
549
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
550
+ you can use `@pypi_base` to set packages required by all
551
+ steps and use `@pypi` to specify step-specific overrides.
634
552
 
635
553
  Parameters
636
554
  ----------
637
- times : int, default 3
638
- Number of times to retry this task.
639
- minutes_between_retries : int, default 2
640
- Number of minutes between retries.
555
+ packages : Dict[str, str], default: {}
556
+ Packages to use for this step. The key is the name of the package
557
+ and the value is the version to use.
558
+ python : str, optional, default: None
559
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
560
+ that the version used will correspond to the version of the Python interpreter used to start the run.
641
561
  """
642
562
  ...
643
563
 
644
564
  @typing.overload
645
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
565
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
646
566
  ...
647
567
 
648
568
  @typing.overload
649
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
650
- ...
651
-
652
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
653
- """
654
- Specifies the number of times the task corresponding
655
- to a step needs to be retried.
656
-
657
- This decorator is useful for handling transient errors, such as networking issues.
658
- If your task contains operations that can't be retried safely, e.g. database updates,
659
- it is advisable to annotate it with `@retry(times=0)`.
660
-
661
- This can be used in conjunction with the `@catch` decorator. The `@catch`
662
- decorator will execute a no-op task after all retries have been exhausted,
663
- ensuring that the flow execution can continue.
664
-
665
- Parameters
666
- ----------
667
- times : int, default 3
668
- Number of times to retry this task.
669
- minutes_between_retries : int, default 2
670
- Number of minutes between retries.
671
- """
569
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
672
570
  ...
673
571
 
674
- @typing.overload
675
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
572
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
676
573
  """
677
- Specifies secrets to be retrieved and injected as environment variables prior to
678
- the execution of a step.
574
+ Specifies the PyPI packages for the step.
679
575
 
680
- Parameters
681
- ----------
682
- sources : List[Union[str, Dict[str, Any]]], default: []
683
- List of secret specs, defining how the secrets are to be retrieved
684
- """
685
- ...
686
-
687
- @typing.overload
688
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
689
- ...
690
-
691
- @typing.overload
692
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
693
- ...
694
-
695
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
696
- """
697
- Specifies secrets to be retrieved and injected as environment variables prior to
698
- the execution of a step.
576
+ Information in this decorator will augment any
577
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
578
+ you can use `@pypi_base` to set packages required by all
579
+ steps and use `@pypi` to specify step-specific overrides.
699
580
 
700
581
  Parameters
701
582
  ----------
702
- sources : List[Union[str, Dict[str, Any]]], default: []
703
- List of secret specs, defining how the secrets are to be retrieved
583
+ packages : Dict[str, str], default: {}
584
+ Packages to use for this step. The key is the name of the package
585
+ and the value is the version to use.
586
+ python : str, optional, default: None
587
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
588
+ that the version used will correspond to the version of the Python interpreter used to start the run.
704
589
  """
705
590
  ...
706
591
 
@@ -735,60 +620,114 @@ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], Non
735
620
  """
736
621
  ...
737
622
 
738
- @typing.overload
739
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
623
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
740
624
  """
741
- Specifies a timeout for your step.
742
-
743
- This decorator is useful if this step may hang indefinitely.
744
-
745
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
746
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
747
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
748
-
749
- Note that all the values specified in parameters are added together so if you specify
750
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
625
+ Specifies that this step should execute on Kubernetes.
751
626
 
752
627
  Parameters
753
628
  ----------
754
- seconds : int, default 0
755
- Number of seconds to wait prior to timing out.
756
- minutes : int, default 0
757
- Number of minutes to wait prior to timing out.
758
- hours : int, default 0
759
- Number of hours to wait prior to timing out.
760
- """
761
- ...
762
-
763
- @typing.overload
764
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
765
- ...
766
-
767
- @typing.overload
768
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
769
- ...
770
-
771
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
629
+ cpu : int, default 1
630
+ Number of CPUs required for this step. If `@resources` is
631
+ also present, the maximum value from all decorators is used.
632
+ memory : int, default 4096
633
+ Memory size (in MB) required for this step. If
634
+ `@resources` is also present, the maximum value from all decorators is
635
+ used.
636
+ disk : int, default 10240
637
+ Disk size (in MB) required for this step. If
638
+ `@resources` is also present, the maximum value from all decorators is
639
+ used.
640
+ image : str, optional, default None
641
+ Docker image to use when launching on Kubernetes. If not specified, and
642
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
643
+ not, a default Docker image mapping to the current version of Python is used.
644
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
645
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
646
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
647
+ Kubernetes service account to use when launching pod in Kubernetes.
648
+ secrets : List[str], optional, default None
649
+ Kubernetes secrets to use when launching pod in Kubernetes. These
650
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
651
+ in Metaflow configuration.
652
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
653
+ Kubernetes namespace to use when launching pod in Kubernetes.
654
+ gpu : int, optional, default None
655
+ Number of GPUs required for this step. A value of zero implies that
656
+ the scheduled node should not have GPUs.
657
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
658
+ The vendor of the GPUs to be used for this step.
659
+ tolerations : List[str], default []
660
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
661
+ Kubernetes tolerations to use when launching pod in Kubernetes.
662
+ use_tmpfs : bool, default False
663
+ This enables an explicit tmpfs mount for this step.
664
+ tmpfs_tempdir : bool, default True
665
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
666
+ tmpfs_size : int, optional, default: None
667
+ The value for the size (in MiB) of the tmpfs mount for this step.
668
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
669
+ memory allocated for this step.
670
+ tmpfs_path : str, optional, default /metaflow_temp
671
+ Path to tmpfs mount for this step.
672
+ persistent_volume_claims : Dict[str, str], optional, default None
673
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
674
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
675
+ shared_memory: int, optional
676
+ Shared memory size (in MiB) required for this step
677
+ port: int, optional
678
+ Port number to specify in the Kubernetes job object
679
+ compute_pool : str, optional, default None
680
+ Compute pool to be used for for this step.
681
+ If not specified, any accessible compute pool within the perimeter is used.
772
682
  """
773
- Specifies a timeout for your step.
683
+ ...
684
+
685
+ @typing.overload
686
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
687
+ """
688
+ Specifies that the step will success under all circumstances.
774
689
 
775
- This decorator is useful if this step may hang indefinitely.
690
+ The decorator will create an optional artifact, specified by `var`, which
691
+ contains the exception raised. You can use it to detect the presence
692
+ of errors, indicating that all happy-path artifacts produced by the step
693
+ are missing.
776
694
 
777
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
778
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
779
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
695
+ Parameters
696
+ ----------
697
+ var : str, optional, default None
698
+ Name of the artifact in which to store the caught exception.
699
+ If not specified, the exception is not stored.
700
+ print_exception : bool, default True
701
+ Determines whether or not the exception is printed to
702
+ stdout when caught.
703
+ """
704
+ ...
705
+
706
+ @typing.overload
707
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
708
+ ...
709
+
710
+ @typing.overload
711
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
712
+ ...
713
+
714
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
715
+ """
716
+ Specifies that the step will success under all circumstances.
780
717
 
781
- Note that all the values specified in parameters are added together so if you specify
782
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
718
+ The decorator will create an optional artifact, specified by `var`, which
719
+ contains the exception raised. You can use it to detect the presence
720
+ of errors, indicating that all happy-path artifacts produced by the step
721
+ are missing.
783
722
 
784
723
  Parameters
785
724
  ----------
786
- seconds : int, default 0
787
- Number of seconds to wait prior to timing out.
788
- minutes : int, default 0
789
- Number of minutes to wait prior to timing out.
790
- hours : int, default 0
791
- Number of hours to wait prior to timing out.
725
+ var : str, optional, default None
726
+ Name of the artifact in which to store the caught exception.
727
+ If not specified, the exception is not stored.
728
+ print_exception : bool, default True
729
+ Determines whether or not the exception is printed to
730
+ stdout when caught.
792
731
  """
793
732
  ...
794
733
 
@@ -870,51 +809,112 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
870
809
  ...
871
810
 
872
811
  @typing.overload
873
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
812
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
874
813
  """
875
- Specifies that the step will success under all circumstances.
814
+ Specifies a timeout for your step.
876
815
 
877
- The decorator will create an optional artifact, specified by `var`, which
878
- contains the exception raised. You can use it to detect the presence
879
- of errors, indicating that all happy-path artifacts produced by the step
880
- are missing.
816
+ This decorator is useful if this step may hang indefinitely.
817
+
818
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
819
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
820
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
821
+
822
+ Note that all the values specified in parameters are added together so if you specify
823
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
881
824
 
882
825
  Parameters
883
826
  ----------
884
- var : str, optional, default None
885
- Name of the artifact in which to store the caught exception.
886
- If not specified, the exception is not stored.
887
- print_exception : bool, default True
888
- Determines whether or not the exception is printed to
889
- stdout when caught.
827
+ seconds : int, default 0
828
+ Number of seconds to wait prior to timing out.
829
+ minutes : int, default 0
830
+ Number of minutes to wait prior to timing out.
831
+ hours : int, default 0
832
+ Number of hours to wait prior to timing out.
890
833
  """
891
834
  ...
892
835
 
893
836
  @typing.overload
894
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
837
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
895
838
  ...
896
839
 
897
840
  @typing.overload
898
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
841
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
899
842
  ...
900
843
 
901
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
844
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
902
845
  """
903
- Specifies that the step will success under all circumstances.
846
+ Specifies a timeout for your step.
904
847
 
905
- The decorator will create an optional artifact, specified by `var`, which
906
- contains the exception raised. You can use it to detect the presence
907
- of errors, indicating that all happy-path artifacts produced by the step
908
- are missing.
848
+ This decorator is useful if this step may hang indefinitely.
849
+
850
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
851
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
852
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
853
+
854
+ Note that all the values specified in parameters are added together so if you specify
855
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
909
856
 
910
857
  Parameters
911
858
  ----------
912
- var : str, optional, default None
913
- Name of the artifact in which to store the caught exception.
914
- If not specified, the exception is not stored.
915
- print_exception : bool, default True
916
- Determines whether or not the exception is printed to
917
- stdout when caught.
859
+ seconds : int, default 0
860
+ Number of seconds to wait prior to timing out.
861
+ minutes : int, default 0
862
+ Number of minutes to wait prior to timing out.
863
+ hours : int, default 0
864
+ Number of hours to wait prior to timing out.
865
+ """
866
+ ...
867
+
868
+ @typing.overload
869
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
870
+ """
871
+ Specifies the number of times the task corresponding
872
+ to a step needs to be retried.
873
+
874
+ This decorator is useful for handling transient errors, such as networking issues.
875
+ If your task contains operations that can't be retried safely, e.g. database updates,
876
+ it is advisable to annotate it with `@retry(times=0)`.
877
+
878
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
879
+ decorator will execute a no-op task after all retries have been exhausted,
880
+ ensuring that the flow execution can continue.
881
+
882
+ Parameters
883
+ ----------
884
+ times : int, default 3
885
+ Number of times to retry this task.
886
+ minutes_between_retries : int, default 2
887
+ Number of minutes between retries.
888
+ """
889
+ ...
890
+
891
+ @typing.overload
892
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
893
+ ...
894
+
895
+ @typing.overload
896
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
897
+ ...
898
+
899
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
900
+ """
901
+ Specifies the number of times the task corresponding
902
+ to a step needs to be retried.
903
+
904
+ This decorator is useful for handling transient errors, such as networking issues.
905
+ If your task contains operations that can't be retried safely, e.g. database updates,
906
+ it is advisable to annotate it with `@retry(times=0)`.
907
+
908
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
909
+ decorator will execute a no-op task after all retries have been exhausted,
910
+ ensuring that the flow execution can continue.
911
+
912
+ Parameters
913
+ ----------
914
+ times : int, default 3
915
+ Number of times to retry this task.
916
+ minutes_between_retries : int, default 2
917
+ Number of minutes between retries.
918
918
  """
919
919
  ...
920
920
 
@@ -992,24 +992,112 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
992
992
  """
993
993
  ...
994
994
 
995
- @typing.overload
996
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
995
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
997
996
  """
998
- Specifies the times when the flow should be run when running on a
999
- production scheduler.
997
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
998
+
999
+ User code call
1000
+ -----------
1001
+ @nim(
1002
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1003
+ backend='managed'
1004
+ )
1005
+
1006
+ Valid backend options
1007
+ ---------------------
1008
+ - 'managed': Outerbounds selects a compute provider based on the model.
1009
+ - 🚧 'dataplane': Run in your account.
1010
+
1011
+ Valid model options
1012
+ ----------------
1013
+ - 'meta/llama3-8b-instruct': 8B parameter model
1014
+ - 'meta/llama3-70b-instruct': 70B parameter model
1015
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1000
1016
 
1001
1017
  Parameters
1002
1018
  ----------
1003
- hourly : bool, default False
1004
- Run the workflow hourly.
1005
- daily : bool, default True
1006
- Run the workflow daily.
1007
- weekly : bool, default False
1008
- Run the workflow weekly.
1009
- cron : str, optional, default None
1010
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1011
- specified by this expression.
1012
- timezone : str, optional, default None
1019
+ models: list[NIM]
1020
+ List of NIM containers running models in sidecars.
1021
+ backend: str
1022
+ Compute provider to run the NIM container.
1023
+ """
1024
+ ...
1025
+
1026
+ @typing.overload
1027
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1028
+ """
1029
+ Specifies the PyPI packages for all steps of the flow.
1030
+
1031
+ Use `@pypi_base` to set common packages required by all
1032
+ steps and use `@pypi` to specify step-specific overrides.
1033
+ Parameters
1034
+ ----------
1035
+ packages : Dict[str, str], default: {}
1036
+ Packages to use for this flow. The key is the name of the package
1037
+ and the value is the version to use.
1038
+ python : str, optional, default: None
1039
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1040
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1041
+ """
1042
+ ...
1043
+
1044
+ @typing.overload
1045
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1046
+ ...
1047
+
1048
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1049
+ """
1050
+ Specifies the PyPI packages for all steps of the flow.
1051
+
1052
+ Use `@pypi_base` to set common packages required by all
1053
+ steps and use `@pypi` to specify step-specific overrides.
1054
+ Parameters
1055
+ ----------
1056
+ packages : Dict[str, str], default: {}
1057
+ Packages to use for this flow. The key is the name of the package
1058
+ and the value is the version to use.
1059
+ python : str, optional, default: None
1060
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1061
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1062
+ """
1063
+ ...
1064
+
1065
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1066
+ """
1067
+ Specifies what flows belong to the same project.
1068
+
1069
+ A project-specific namespace is created for all flows that
1070
+ use the same `@project(name)`.
1071
+
1072
+ Parameters
1073
+ ----------
1074
+ name : str
1075
+ Project name. Make sure that the name is unique amongst all
1076
+ projects that use the same production scheduler. The name may
1077
+ contain only lowercase alphanumeric characters and underscores.
1078
+
1079
+
1080
+ """
1081
+ ...
1082
+
1083
+ @typing.overload
1084
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1085
+ """
1086
+ Specifies the times when the flow should be run when running on a
1087
+ production scheduler.
1088
+
1089
+ Parameters
1090
+ ----------
1091
+ hourly : bool, default False
1092
+ Run the workflow hourly.
1093
+ daily : bool, default True
1094
+ Run the workflow daily.
1095
+ weekly : bool, default False
1096
+ Run the workflow weekly.
1097
+ cron : str, optional, default None
1098
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1099
+ specified by this expression.
1100
+ timezone : str, optional, default None
1013
1101
  Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1014
1102
  which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1015
1103
  """
@@ -1090,67 +1178,87 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1090
1178
  """
1091
1179
  ...
1092
1180
 
1093
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1181
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1094
1182
  """
1095
- Specifies what flows belong to the same project.
1096
-
1097
- A project-specific namespace is created for all flows that
1098
- use the same `@project(name)`.
1183
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1184
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1185
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1186
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1187
+ starts only after all sensors finish.
1099
1188
 
1100
1189
  Parameters
1101
1190
  ----------
1191
+ timeout : int
1192
+ Time, in seconds before the task times out and fails. (Default: 3600)
1193
+ poke_interval : int
1194
+ Time in seconds that the job should wait in between each try. (Default: 60)
1195
+ mode : str
1196
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1197
+ exponential_backoff : bool
1198
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1199
+ pool : str
1200
+ the slot pool this task should run in,
1201
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1202
+ soft_fail : bool
1203
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1102
1204
  name : str
1103
- Project name. Make sure that the name is unique amongst all
1104
- projects that use the same production scheduler. The name may
1105
- contain only lowercase alphanumeric characters and underscores.
1106
-
1107
-
1205
+ Name of the sensor on Airflow
1206
+ description : str
1207
+ Description of sensor in the Airflow UI
1208
+ bucket_key : Union[str, List[str]]
1209
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1210
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1211
+ bucket_name : str
1212
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1213
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1214
+ wildcard_match : bool
1215
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1216
+ aws_conn_id : str
1217
+ a reference to the s3 connection on Airflow. (Default: None)
1218
+ verify : bool
1219
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1108
1220
  """
1109
1221
  ...
1110
1222
 
1111
1223
  @typing.overload
1112
- def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1224
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1113
1225
  """
1114
- Specifies the flow(s) that this flow depends on.
1226
+ Specifies the event(s) that this flow depends on.
1115
1227
 
1116
1228
  ```
1117
- @trigger_on_finish(flow='FooFlow')
1229
+ @trigger(event='foo')
1118
1230
  ```
1119
1231
  or
1120
1232
  ```
1121
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1233
+ @trigger(events=['foo', 'bar'])
1122
1234
  ```
1123
- This decorator respects the @project decorator and triggers the flow
1124
- when upstream runs within the same namespace complete successfully
1125
1235
 
1126
- Additionally, you can specify project aware upstream flow dependencies
1127
- by specifying the fully qualified project_flow_name.
1236
+ Additionally, you can specify the parameter mappings
1237
+ to map event payload to Metaflow parameters for the flow.
1128
1238
  ```
1129
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1239
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1130
1240
  ```
1131
1241
  or
1132
1242
  ```
1133
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1243
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1244
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1134
1245
  ```
1135
1246
 
1136
- You can also specify just the project or project branch (other values will be
1137
- inferred from the current project or project branch):
1247
+ 'parameters' can also be a list of strings and tuples like so:
1138
1248
  ```
1139
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1249
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1250
+ ```
1251
+ This is equivalent to:
1252
+ ```
1253
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1140
1254
  ```
1141
-
1142
- Note that `branch` is typically one of:
1143
- - `prod`
1144
- - `user.bob`
1145
- - `test.my_experiment`
1146
- - `prod.staging`
1147
1255
 
1148
1256
  Parameters
1149
1257
  ----------
1150
- flow : Union[str, Dict[str, str]], optional, default None
1151
- Upstream flow dependency for this flow.
1152
- flows : List[Union[str, Dict[str, str]]], default []
1153
- Upstream flow dependencies for this flow.
1258
+ event : Union[str, Dict[str, Any]], optional, default None
1259
+ Event dependency for this flow.
1260
+ events : List[Union[str, Dict[str, Any]]], default []
1261
+ Events dependency for this flow.
1154
1262
  options : Dict[str, Any], default {}
1155
1263
  Backend-specific configuration for tuning eventing behavior.
1156
1264
 
@@ -1159,51 +1267,47 @@ def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] =
1159
1267
  ...
1160
1268
 
1161
1269
  @typing.overload
1162
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1270
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1163
1271
  ...
1164
1272
 
1165
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1273
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1166
1274
  """
1167
- Specifies the flow(s) that this flow depends on.
1275
+ Specifies the event(s) that this flow depends on.
1168
1276
 
1169
1277
  ```
1170
- @trigger_on_finish(flow='FooFlow')
1278
+ @trigger(event='foo')
1171
1279
  ```
1172
1280
  or
1173
1281
  ```
1174
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1282
+ @trigger(events=['foo', 'bar'])
1175
1283
  ```
1176
- This decorator respects the @project decorator and triggers the flow
1177
- when upstream runs within the same namespace complete successfully
1178
1284
 
1179
- Additionally, you can specify project aware upstream flow dependencies
1180
- by specifying the fully qualified project_flow_name.
1285
+ Additionally, you can specify the parameter mappings
1286
+ to map event payload to Metaflow parameters for the flow.
1181
1287
  ```
1182
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1288
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1183
1289
  ```
1184
1290
  or
1185
1291
  ```
1186
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1292
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1293
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1187
1294
  ```
1188
1295
 
1189
- You can also specify just the project or project branch (other values will be
1190
- inferred from the current project or project branch):
1296
+ 'parameters' can also be a list of strings and tuples like so:
1191
1297
  ```
1192
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1298
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1299
+ ```
1300
+ This is equivalent to:
1301
+ ```
1302
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1193
1303
  ```
1194
-
1195
- Note that `branch` is typically one of:
1196
- - `prod`
1197
- - `user.bob`
1198
- - `test.my_experiment`
1199
- - `prod.staging`
1200
1304
 
1201
1305
  Parameters
1202
1306
  ----------
1203
- flow : Union[str, Dict[str, str]], optional, default None
1204
- Upstream flow dependency for this flow.
1205
- flows : List[Union[str, Dict[str, str]]], default []
1206
- Upstream flow dependencies for this flow.
1307
+ event : Union[str, Dict[str, Any]], optional, default None
1308
+ Event dependency for this flow.
1309
+ events : List[Union[str, Dict[str, Any]]], default []
1310
+ Events dependency for this flow.
1207
1311
  options : Dict[str, Any], default {}
1208
1312
  Backend-specific configuration for tuning eventing behavior.
1209
1313
 
@@ -1211,118 +1315,6 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1211
1315
  """
1212
1316
  ...
1213
1317
 
1214
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1215
- """
1216
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1217
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1218
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1219
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1220
- starts only after all sensors finish.
1221
-
1222
- Parameters
1223
- ----------
1224
- timeout : int
1225
- Time, in seconds before the task times out and fails. (Default: 3600)
1226
- poke_interval : int
1227
- Time in seconds that the job should wait in between each try. (Default: 60)
1228
- mode : str
1229
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1230
- exponential_backoff : bool
1231
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1232
- pool : str
1233
- the slot pool this task should run in,
1234
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1235
- soft_fail : bool
1236
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1237
- name : str
1238
- Name of the sensor on Airflow
1239
- description : str
1240
- Description of sensor in the Airflow UI
1241
- bucket_key : Union[str, List[str]]
1242
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1243
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1244
- bucket_name : str
1245
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1246
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1247
- wildcard_match : bool
1248
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1249
- aws_conn_id : str
1250
- a reference to the s3 connection on Airflow. (Default: None)
1251
- verify : bool
1252
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1253
- """
1254
- ...
1255
-
1256
- @typing.overload
1257
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1258
- """
1259
- Specifies the PyPI packages for all steps of the flow.
1260
-
1261
- Use `@pypi_base` to set common packages required by all
1262
- steps and use `@pypi` to specify step-specific overrides.
1263
- Parameters
1264
- ----------
1265
- packages : Dict[str, str], default: {}
1266
- Packages to use for this flow. The key is the name of the package
1267
- and the value is the version to use.
1268
- python : str, optional, default: None
1269
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1270
- that the version used will correspond to the version of the Python interpreter used to start the run.
1271
- """
1272
- ...
1273
-
1274
- @typing.overload
1275
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1276
- ...
1277
-
1278
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1279
- """
1280
- Specifies the PyPI packages for all steps of the flow.
1281
-
1282
- Use `@pypi_base` to set common packages required by all
1283
- steps and use `@pypi` to specify step-specific overrides.
1284
- Parameters
1285
- ----------
1286
- packages : Dict[str, str], default: {}
1287
- Packages to use for this flow. The key is the name of the package
1288
- and the value is the version to use.
1289
- python : str, optional, default: None
1290
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1291
- that the version used will correspond to the version of the Python interpreter used to start the run.
1292
- """
1293
- ...
1294
-
1295
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1296
- """
1297
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1298
-
1299
- User code call
1300
- -----------
1301
- @nim(
1302
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1303
- backend='managed'
1304
- )
1305
-
1306
- Valid backend options
1307
- ---------------------
1308
- - 'managed': Outerbounds selects a compute provider based on the model.
1309
- - 🚧 'dataplane': Run in your account.
1310
-
1311
- Valid model options
1312
- ----------------
1313
- - 'meta/llama3-8b-instruct': 8B parameter model
1314
- - 'meta/llama3-70b-instruct': 70B parameter model
1315
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1316
-
1317
- Parameters
1318
- ----------
1319
- models: list[NIM]
1320
- List of NIM containers running models in sidecars.
1321
- backend: str
1322
- Compute provider to run the NIM container.
1323
- """
1324
- ...
1325
-
1326
1318
  def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1327
1319
  """
1328
1320
  The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
@@ -1366,44 +1358,48 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1366
1358
  ...
1367
1359
 
1368
1360
  @typing.overload
1369
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1361
+ def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1370
1362
  """
1371
- Specifies the event(s) that this flow depends on.
1363
+ Specifies the flow(s) that this flow depends on.
1372
1364
 
1373
1365
  ```
1374
- @trigger(event='foo')
1366
+ @trigger_on_finish(flow='FooFlow')
1375
1367
  ```
1376
1368
  or
1377
1369
  ```
1378
- @trigger(events=['foo', 'bar'])
1370
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1379
1371
  ```
1372
+ This decorator respects the @project decorator and triggers the flow
1373
+ when upstream runs within the same namespace complete successfully
1380
1374
 
1381
- Additionally, you can specify the parameter mappings
1382
- to map event payload to Metaflow parameters for the flow.
1375
+ Additionally, you can specify project aware upstream flow dependencies
1376
+ by specifying the fully qualified project_flow_name.
1383
1377
  ```
1384
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1378
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1385
1379
  ```
1386
1380
  or
1387
1381
  ```
1388
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1389
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1382
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1390
1383
  ```
1391
1384
 
1392
- 'parameters' can also be a list of strings and tuples like so:
1393
- ```
1394
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1395
- ```
1396
- This is equivalent to:
1385
+ You can also specify just the project or project branch (other values will be
1386
+ inferred from the current project or project branch):
1397
1387
  ```
1398
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1388
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1399
1389
  ```
1400
1390
 
1391
+ Note that `branch` is typically one of:
1392
+ - `prod`
1393
+ - `user.bob`
1394
+ - `test.my_experiment`
1395
+ - `prod.staging`
1396
+
1401
1397
  Parameters
1402
1398
  ----------
1403
- event : Union[str, Dict[str, Any]], optional, default None
1404
- Event dependency for this flow.
1405
- events : List[Union[str, Dict[str, Any]]], default []
1406
- Events dependency for this flow.
1399
+ flow : Union[str, Dict[str, str]], optional, default None
1400
+ Upstream flow dependency for this flow.
1401
+ flows : List[Union[str, Dict[str, str]]], default []
1402
+ Upstream flow dependencies for this flow.
1407
1403
  options : Dict[str, Any], default {}
1408
1404
  Backend-specific configuration for tuning eventing behavior.
1409
1405
 
@@ -1412,47 +1408,51 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
1412
1408
  ...
1413
1409
 
1414
1410
  @typing.overload
1415
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1411
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1416
1412
  ...
1417
1413
 
1418
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1414
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1419
1415
  """
1420
- Specifies the event(s) that this flow depends on.
1416
+ Specifies the flow(s) that this flow depends on.
1421
1417
 
1422
1418
  ```
1423
- @trigger(event='foo')
1419
+ @trigger_on_finish(flow='FooFlow')
1424
1420
  ```
1425
1421
  or
1426
1422
  ```
1427
- @trigger(events=['foo', 'bar'])
1423
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1428
1424
  ```
1425
+ This decorator respects the @project decorator and triggers the flow
1426
+ when upstream runs within the same namespace complete successfully
1429
1427
 
1430
- Additionally, you can specify the parameter mappings
1431
- to map event payload to Metaflow parameters for the flow.
1428
+ Additionally, you can specify project aware upstream flow dependencies
1429
+ by specifying the fully qualified project_flow_name.
1432
1430
  ```
1433
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1431
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1434
1432
  ```
1435
1433
  or
1436
1434
  ```
1437
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1438
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1435
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1439
1436
  ```
1440
1437
 
1441
- 'parameters' can also be a list of strings and tuples like so:
1442
- ```
1443
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1444
- ```
1445
- This is equivalent to:
1438
+ You can also specify just the project or project branch (other values will be
1439
+ inferred from the current project or project branch):
1446
1440
  ```
1447
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1441
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1448
1442
  ```
1449
1443
 
1444
+ Note that `branch` is typically one of:
1445
+ - `prod`
1446
+ - `user.bob`
1447
+ - `test.my_experiment`
1448
+ - `prod.staging`
1449
+
1450
1450
  Parameters
1451
1451
  ----------
1452
- event : Union[str, Dict[str, Any]], optional, default None
1453
- Event dependency for this flow.
1454
- events : List[Union[str, Dict[str, Any]]], default []
1455
- Events dependency for this flow.
1452
+ flow : Union[str, Dict[str, str]], optional, default None
1453
+ Upstream flow dependency for this flow.
1454
+ flows : List[Union[str, Dict[str, str]]], default []
1455
+ Upstream flow dependencies for this flow.
1456
1456
  options : Dict[str, Any], default {}
1457
1457
  Backend-specific configuration for tuning eventing behavior.
1458
1458