ob-metaflow-stubs 5.3__py2.py3-none-any.whl → 5.5__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. metaflow-stubs/__init__.pyi +478 -478
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +2 -2
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +4 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +3 -3
  13. metaflow-stubs/metadata/metadata.pyi +3 -3
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +34 -34
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +3 -3
  20. metaflow-stubs/plugins/__init__.pyi +2 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  27. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  28. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  29. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  32. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
  33. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +6 -6
  34. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
  35. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  38. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  42. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  43. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
  44. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  45. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
  51. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  52. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
  55. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  56. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  57. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  58. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  59. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  60. metaflow-stubs/plugins/cards/card_client.pyi +2 -2
  61. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  62. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  63. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/basic.pyi +4 -4
  66. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  68. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  71. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  72. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  73. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  74. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  75. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  76. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  77. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  78. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  79. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  80. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  81. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  82. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  83. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  84. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  85. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  87. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  88. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  90. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  93. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  95. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +4 -4
  96. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  98. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  99. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  100. metaflow-stubs/plugins/package_cli.pyi +2 -2
  101. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  102. metaflow-stubs/plugins/perimeters.pyi +2 -2
  103. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  104. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  105. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  107. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  109. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  110. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  111. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  112. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  113. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  114. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  116. metaflow-stubs/plugins/tag_cli.pyi +4 -4
  117. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  119. metaflow-stubs/procpoll.pyi +2 -2
  120. metaflow-stubs/profilers/__init__.pyi +2 -2
  121. metaflow-stubs/pylint_wrapper.pyi +2 -2
  122. metaflow-stubs/runner/__init__.pyi +2 -2
  123. metaflow-stubs/runner/deployer.pyi +3 -3
  124. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  125. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  126. metaflow-stubs/runner/nbrun.pyi +2 -2
  127. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  128. metaflow-stubs/runner/utils.pyi +2 -2
  129. metaflow-stubs/system/__init__.pyi +3 -3
  130. metaflow-stubs/system/system_logger.pyi +2 -2
  131. metaflow-stubs/system/system_monitor.pyi +2 -2
  132. metaflow-stubs/tagging_util.pyi +2 -2
  133. metaflow-stubs/tuple_util.pyi +2 -2
  134. {ob_metaflow_stubs-5.3.dist-info → ob_metaflow_stubs-5.5.dist-info}/METADATA +1 -1
  135. ob_metaflow_stubs-5.5.dist-info/RECORD +138 -0
  136. ob_metaflow_stubs-5.3.dist-info/RECORD +0 -138
  137. {ob_metaflow_stubs-5.3.dist-info → ob_metaflow_stubs-5.5.dist-info}/WHEEL +0 -0
  138. {ob_metaflow_stubs-5.3.dist-info → ob_metaflow_stubs-5.5.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.15.1+ob(v1) #
4
- # Generated on 2024-08-22T21:03:01.630360 #
3
+ # MF version: 2.12.17.1+ob(v1) #
4
+ # Generated on 2024-08-27T02:02:03.052211 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.flowspec
12
- import metaflow.parameters
13
- import metaflow.plugins.datatools.s3.s3
11
+ import metaflow.datastore.inputs
14
12
  import io
15
- import datetime
16
13
  import metaflow.events
17
- import metaflow.datastore.inputs
14
+ import metaflow.plugins.datatools.s3.s3
18
15
  import metaflow.metaflow_current
19
- import metaflow.client.core
20
16
  import metaflow.runner.metaflow_runner
17
+ import metaflow.flowspec
18
+ import datetime
21
19
  import metaflow._vendor.click.types
20
+ import metaflow.parameters
21
+ import metaflow.client.core
22
22
  import typing
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
@@ -727,112 +727,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
727
727
  """
728
728
  ...
729
729
 
730
- @typing.overload
731
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
- """
733
- Specifies that the step will success under all circumstances.
734
-
735
- The decorator will create an optional artifact, specified by `var`, which
736
- contains the exception raised. You can use it to detect the presence
737
- of errors, indicating that all happy-path artifacts produced by the step
738
- are missing.
739
-
740
- Parameters
741
- ----------
742
- var : str, optional, default None
743
- Name of the artifact in which to store the caught exception.
744
- If not specified, the exception is not stored.
745
- print_exception : bool, default True
746
- Determines whether or not the exception is printed to
747
- stdout when caught.
748
- """
749
- ...
750
-
751
- @typing.overload
752
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
753
- ...
754
-
755
- @typing.overload
756
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
757
- ...
758
-
759
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
760
- """
761
- Specifies that the step will success under all circumstances.
762
-
763
- The decorator will create an optional artifact, specified by `var`, which
764
- contains the exception raised. You can use it to detect the presence
765
- of errors, indicating that all happy-path artifacts produced by the step
766
- are missing.
767
-
768
- Parameters
769
- ----------
770
- var : str, optional, default None
771
- Name of the artifact in which to store the caught exception.
772
- If not specified, the exception is not stored.
773
- print_exception : bool, default True
774
- Determines whether or not the exception is printed to
775
- stdout when caught.
776
- """
777
- ...
778
-
779
- @typing.overload
780
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
781
- """
782
- Specifies a timeout for your step.
783
-
784
- This decorator is useful if this step may hang indefinitely.
785
-
786
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
787
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
788
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
789
-
790
- Note that all the values specified in parameters are added together so if you specify
791
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
792
-
793
- Parameters
794
- ----------
795
- seconds : int, default 0
796
- Number of seconds to wait prior to timing out.
797
- minutes : int, default 0
798
- Number of minutes to wait prior to timing out.
799
- hours : int, default 0
800
- Number of hours to wait prior to timing out.
801
- """
802
- ...
803
-
804
- @typing.overload
805
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
806
- ...
807
-
808
- @typing.overload
809
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
810
- ...
811
-
812
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
813
- """
814
- Specifies a timeout for your step.
815
-
816
- This decorator is useful if this step may hang indefinitely.
817
-
818
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
819
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
820
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
821
-
822
- Note that all the values specified in parameters are added together so if you specify
823
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
824
-
825
- Parameters
826
- ----------
827
- seconds : int, default 0
828
- Number of seconds to wait prior to timing out.
829
- minutes : int, default 0
830
- Number of minutes to wait prior to timing out.
831
- hours : int, default 0
832
- Number of hours to wait prior to timing out.
833
- """
834
- ...
835
-
836
730
  @typing.overload
837
731
  def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
838
732
  """
@@ -910,84 +804,71 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
910
804
  """
911
805
  ...
912
806
 
913
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
807
+ @typing.overload
808
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
914
809
  """
915
- Specifies that this step should execute on Kubernetes.
810
+ Internal decorator to support Fast bakery
811
+ """
812
+ ...
813
+
814
+ @typing.overload
815
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
816
+ ...
817
+
818
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
819
+ """
820
+ Internal decorator to support Fast bakery
821
+ """
822
+ ...
823
+
824
+ @typing.overload
825
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
826
+ """
827
+ Creates a human-readable report, a Metaflow Card, after this step completes.
828
+
829
+ Note that you may add multiple `@card` decorators in a step with different parameters.
916
830
 
917
831
  Parameters
918
832
  ----------
919
- cpu : int, default 1
920
- Number of CPUs required for this step. If `@resources` is
921
- also present, the maximum value from all decorators is used.
922
- memory : int, default 4096
923
- Memory size (in MB) required for this step. If
924
- `@resources` is also present, the maximum value from all decorators is
925
- used.
926
- disk : int, default 10240
927
- Disk size (in MB) required for this step. If
928
- `@resources` is also present, the maximum value from all decorators is
929
- used.
930
- image : str, optional, default None
931
- Docker image to use when launching on Kubernetes. If not specified, and
932
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
933
- not, a default Docker image mapping to the current version of Python is used.
934
- image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
935
- If given, the imagePullPolicy to be applied to the Docker image of the step.
936
- service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
937
- Kubernetes service account to use when launching pod in Kubernetes.
938
- secrets : List[str], optional, default None
939
- Kubernetes secrets to use when launching pod in Kubernetes. These
940
- secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
941
- in Metaflow configuration.
942
- namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
943
- Kubernetes namespace to use when launching pod in Kubernetes.
944
- gpu : int, optional, default None
945
- Number of GPUs required for this step. A value of zero implies that
946
- the scheduled node should not have GPUs.
947
- gpu_vendor : str, default KUBERNETES_GPU_VENDOR
948
- The vendor of the GPUs to be used for this step.
949
- tolerations : List[str], default []
950
- The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
951
- Kubernetes tolerations to use when launching pod in Kubernetes.
952
- use_tmpfs : bool, default False
953
- This enables an explicit tmpfs mount for this step.
954
- tmpfs_tempdir : bool, default True
955
- sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
956
- tmpfs_size : int, optional, default: None
957
- The value for the size (in MiB) of the tmpfs mount for this step.
958
- This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
959
- memory allocated for this step.
960
- tmpfs_path : str, optional, default /metaflow_temp
961
- Path to tmpfs mount for this step.
962
- persistent_volume_claims : Dict[str, str], optional, default None
963
- A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
964
- volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
965
- shared_memory: int, optional
966
- Shared memory size (in MiB) required for this step
967
- port: int, optional
968
- Port number to specify in the Kubernetes job object
969
- compute_pool : str, optional, default None
970
- Compute pool to be used for for this step.
971
- If not specified, any accessible compute pool within the perimeter is used.
833
+ type : str, default 'default'
834
+ Card type.
835
+ id : str, optional, default None
836
+ If multiple cards are present, use this id to identify this card.
837
+ options : Dict[str, Any], default {}
838
+ Options passed to the card. The contents depend on the card type.
839
+ timeout : int, default 45
840
+ Interrupt reporting if it takes more than this many seconds.
841
+
842
+
972
843
  """
973
844
  ...
974
845
 
975
846
  @typing.overload
976
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
977
- """
978
- Decorator prototype for all step decorators. This function gets specialized
979
- and imported for all decorators types by _import_plugin_decorators().
980
- """
847
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
981
848
  ...
982
849
 
983
850
  @typing.overload
984
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
851
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
985
852
  ...
986
853
 
987
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
854
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
988
855
  """
989
- Decorator prototype for all step decorators. This function gets specialized
990
- and imported for all decorators types by _import_plugin_decorators().
856
+ Creates a human-readable report, a Metaflow Card, after this step completes.
857
+
858
+ Note that you may add multiple `@card` decorators in a step with different parameters.
859
+
860
+ Parameters
861
+ ----------
862
+ type : str, default 'default'
863
+ Card type.
864
+ id : str, optional, default None
865
+ If multiple cards are present, use this id to identify this card.
866
+ options : Dict[str, Any], default {}
867
+ Options passed to the card. The contents depend on the card type.
868
+ timeout : int, default 45
869
+ Interrupt reporting if it takes more than this many seconds.
870
+
871
+
991
872
  """
992
873
  ...
993
874
 
@@ -1025,156 +906,165 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
1025
906
  ...
1026
907
 
1027
908
  @typing.overload
1028
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
909
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1029
910
  """
1030
- Specifies the Conda environment for the step.
911
+ Specifies that the step will success under all circumstances.
1031
912
 
1032
- Information in this decorator will augment any
1033
- attributes set in the `@conda_base` flow-level decorator. Hence,
1034
- you can use `@conda_base` to set packages required by all
1035
- steps and use `@conda` to specify step-specific overrides.
913
+ The decorator will create an optional artifact, specified by `var`, which
914
+ contains the exception raised. You can use it to detect the presence
915
+ of errors, indicating that all happy-path artifacts produced by the step
916
+ are missing.
1036
917
 
1037
918
  Parameters
1038
919
  ----------
1039
- packages : Dict[str, str], default {}
1040
- Packages to use for this step. The key is the name of the package
1041
- and the value is the version to use.
1042
- libraries : Dict[str, str], default {}
1043
- Supported for backward compatibility. When used with packages, packages will take precedence.
1044
- python : str, optional, default None
1045
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1046
- that the version used will correspond to the version of the Python interpreter used to start the run.
1047
- disabled : bool, default False
1048
- If set to True, disables @conda.
1049
- """
920
+ var : str, optional, default None
921
+ Name of the artifact in which to store the caught exception.
922
+ If not specified, the exception is not stored.
923
+ print_exception : bool, default True
924
+ Determines whether or not the exception is printed to
925
+ stdout when caught.
926
+ """
1050
927
  ...
1051
928
 
1052
929
  @typing.overload
1053
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
930
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1054
931
  ...
1055
932
 
1056
933
  @typing.overload
1057
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
934
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1058
935
  ...
1059
936
 
1060
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
937
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
1061
938
  """
1062
- Specifies the Conda environment for the step.
939
+ Specifies that the step will success under all circumstances.
1063
940
 
1064
- Information in this decorator will augment any
1065
- attributes set in the `@conda_base` flow-level decorator. Hence,
1066
- you can use `@conda_base` to set packages required by all
1067
- steps and use `@conda` to specify step-specific overrides.
941
+ The decorator will create an optional artifact, specified by `var`, which
942
+ contains the exception raised. You can use it to detect the presence
943
+ of errors, indicating that all happy-path artifacts produced by the step
944
+ are missing.
1068
945
 
1069
946
  Parameters
1070
947
  ----------
1071
- packages : Dict[str, str], default {}
1072
- Packages to use for this step. The key is the name of the package
1073
- and the value is the version to use.
1074
- libraries : Dict[str, str], default {}
1075
- Supported for backward compatibility. When used with packages, packages will take precedence.
1076
- python : str, optional, default None
1077
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1078
- that the version used will correspond to the version of the Python interpreter used to start the run.
1079
- disabled : bool, default False
1080
- If set to True, disables @conda.
948
+ var : str, optional, default None
949
+ Name of the artifact in which to store the caught exception.
950
+ If not specified, the exception is not stored.
951
+ print_exception : bool, default True
952
+ Determines whether or not the exception is printed to
953
+ stdout when caught.
1081
954
  """
1082
955
  ...
1083
956
 
1084
957
  @typing.overload
1085
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
958
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1086
959
  """
1087
- Specifies environment variables to be set prior to the execution of a step.
960
+ Specifies a timeout for your step.
961
+
962
+ This decorator is useful if this step may hang indefinitely.
963
+
964
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
965
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
966
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
967
+
968
+ Note that all the values specified in parameters are added together so if you specify
969
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1088
970
 
1089
971
  Parameters
1090
972
  ----------
1091
- vars : Dict[str, str], default {}
1092
- Dictionary of environment variables to set.
973
+ seconds : int, default 0
974
+ Number of seconds to wait prior to timing out.
975
+ minutes : int, default 0
976
+ Number of minutes to wait prior to timing out.
977
+ hours : int, default 0
978
+ Number of hours to wait prior to timing out.
1093
979
  """
1094
980
  ...
1095
981
 
1096
982
  @typing.overload
1097
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
983
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1098
984
  ...
1099
985
 
1100
986
  @typing.overload
1101
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
987
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1102
988
  ...
1103
989
 
1104
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
990
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1105
991
  """
1106
- Specifies environment variables to be set prior to the execution of a step.
992
+ Specifies a timeout for your step.
993
+
994
+ This decorator is useful if this step may hang indefinitely.
995
+
996
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
997
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
998
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
999
+
1000
+ Note that all the values specified in parameters are added together so if you specify
1001
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1107
1002
 
1108
1003
  Parameters
1109
1004
  ----------
1110
- vars : Dict[str, str], default {}
1111
- Dictionary of environment variables to set.
1005
+ seconds : int, default 0
1006
+ Number of seconds to wait prior to timing out.
1007
+ minutes : int, default 0
1008
+ Number of minutes to wait prior to timing out.
1009
+ hours : int, default 0
1010
+ Number of hours to wait prior to timing out.
1112
1011
  """
1113
1012
  ...
1114
1013
 
1115
1014
  @typing.overload
1116
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1015
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1117
1016
  """
1118
- Specifies the PyPI packages for the step.
1017
+ Specifies the Conda environment for the step.
1119
1018
 
1120
1019
  Information in this decorator will augment any
1121
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1122
- you can use `@pypi_base` to set packages required by all
1123
- steps and use `@pypi` to specify step-specific overrides.
1020
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1021
+ you can use `@conda_base` to set packages required by all
1022
+ steps and use `@conda` to specify step-specific overrides.
1124
1023
 
1125
1024
  Parameters
1126
1025
  ----------
1127
- packages : Dict[str, str], default: {}
1026
+ packages : Dict[str, str], default {}
1128
1027
  Packages to use for this step. The key is the name of the package
1129
1028
  and the value is the version to use.
1130
- python : str, optional, default: None
1029
+ libraries : Dict[str, str], default {}
1030
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1031
+ python : str, optional, default None
1131
1032
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1132
1033
  that the version used will correspond to the version of the Python interpreter used to start the run.
1034
+ disabled : bool, default False
1035
+ If set to True, disables @conda.
1133
1036
  """
1134
1037
  ...
1135
1038
 
1136
1039
  @typing.overload
1137
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1040
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1138
1041
  ...
1139
1042
 
1140
1043
  @typing.overload
1141
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1044
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1142
1045
  ...
1143
1046
 
1144
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1047
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1145
1048
  """
1146
- Specifies the PyPI packages for the step.
1049
+ Specifies the Conda environment for the step.
1147
1050
 
1148
1051
  Information in this decorator will augment any
1149
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1150
- you can use `@pypi_base` to set packages required by all
1151
- steps and use `@pypi` to specify step-specific overrides.
1052
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1053
+ you can use `@conda_base` to set packages required by all
1054
+ steps and use `@conda` to specify step-specific overrides.
1152
1055
 
1153
1056
  Parameters
1154
1057
  ----------
1155
- packages : Dict[str, str], default: {}
1058
+ packages : Dict[str, str], default {}
1156
1059
  Packages to use for this step. The key is the name of the package
1157
1060
  and the value is the version to use.
1158
- python : str, optional, default: None
1061
+ libraries : Dict[str, str], default {}
1062
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1063
+ python : str, optional, default None
1159
1064
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1160
1065
  that the version used will correspond to the version of the Python interpreter used to start the run.
1161
- """
1162
- ...
1163
-
1164
- @typing.overload
1165
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1166
- """
1167
- Internal decorator to support Fast bakery
1168
- """
1169
- ...
1170
-
1171
- @typing.overload
1172
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1173
- ...
1174
-
1175
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1176
- """
1177
- Internal decorator to support Fast bakery
1066
+ disabled : bool, default False
1067
+ If set to True, disables @conda.
1178
1068
  """
1179
1069
  ...
1180
1070
 
@@ -1231,188 +1121,252 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1231
1121
  """
1232
1122
  ...
1233
1123
 
1234
- @typing.overload
1235
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1124
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1236
1125
  """
1237
- Creates a human-readable report, a Metaflow Card, after this step completes.
1238
-
1239
- Note that you may add multiple `@card` decorators in a step with different parameters.
1126
+ Specifies that this step should execute on Kubernetes.
1240
1127
 
1241
1128
  Parameters
1242
1129
  ----------
1243
- type : str, default 'default'
1244
- Card type.
1245
- id : str, optional, default None
1246
- If multiple cards are present, use this id to identify this card.
1247
- options : Dict[str, Any], default {}
1248
- Options passed to the card. The contents depend on the card type.
1249
- timeout : int, default 45
1250
- Interrupt reporting if it takes more than this many seconds.
1251
-
1130
+ cpu : int, default 1
1131
+ Number of CPUs required for this step. If `@resources` is
1132
+ also present, the maximum value from all decorators is used.
1133
+ memory : int, default 4096
1134
+ Memory size (in MB) required for this step. If
1135
+ `@resources` is also present, the maximum value from all decorators is
1136
+ used.
1137
+ disk : int, default 10240
1138
+ Disk size (in MB) required for this step. If
1139
+ `@resources` is also present, the maximum value from all decorators is
1140
+ used.
1141
+ image : str, optional, default None
1142
+ Docker image to use when launching on Kubernetes. If not specified, and
1143
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1144
+ not, a default Docker image mapping to the current version of Python is used.
1145
+ image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
1146
+ If given, the imagePullPolicy to be applied to the Docker image of the step.
1147
+ service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
1148
+ Kubernetes service account to use when launching pod in Kubernetes.
1149
+ secrets : List[str], optional, default None
1150
+ Kubernetes secrets to use when launching pod in Kubernetes. These
1151
+ secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
1152
+ in Metaflow configuration.
1153
+ namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
1154
+ Kubernetes namespace to use when launching pod in Kubernetes.
1155
+ gpu : int, optional, default None
1156
+ Number of GPUs required for this step. A value of zero implies that
1157
+ the scheduled node should not have GPUs.
1158
+ gpu_vendor : str, default KUBERNETES_GPU_VENDOR
1159
+ The vendor of the GPUs to be used for this step.
1160
+ tolerations : List[str], default []
1161
+ The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
1162
+ Kubernetes tolerations to use when launching pod in Kubernetes.
1163
+ use_tmpfs : bool, default False
1164
+ This enables an explicit tmpfs mount for this step.
1165
+ tmpfs_tempdir : bool, default True
1166
+ sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
1167
+ tmpfs_size : int, optional, default: None
1168
+ The value for the size (in MiB) of the tmpfs mount for this step.
1169
+ This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
1170
+ memory allocated for this step.
1171
+ tmpfs_path : str, optional, default /metaflow_temp
1172
+ Path to tmpfs mount for this step.
1173
+ persistent_volume_claims : Dict[str, str], optional, default None
1174
+ A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
1175
+ volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
1176
+ shared_memory: int, optional
1177
+ Shared memory size (in MiB) required for this step
1178
+ port: int, optional
1179
+ Port number to specify in the Kubernetes job object
1180
+ compute_pool : str, optional, default None
1181
+ Compute pool to be used for for this step.
1182
+ If not specified, any accessible compute pool within the perimeter is used.
1183
+ """
1184
+ ...
1185
+
1186
+ @typing.overload
1187
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1188
+ """
1189
+ Specifies environment variables to be set prior to the execution of a step.
1252
1190
 
1191
+ Parameters
1192
+ ----------
1193
+ vars : Dict[str, str], default {}
1194
+ Dictionary of environment variables to set.
1253
1195
  """
1254
1196
  ...
1255
1197
 
1256
1198
  @typing.overload
1257
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1199
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1258
1200
  ...
1259
1201
 
1260
1202
  @typing.overload
1261
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1203
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1262
1204
  ...
1263
1205
 
1264
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1206
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1265
1207
  """
1266
- Creates a human-readable report, a Metaflow Card, after this step completes.
1267
-
1268
- Note that you may add multiple `@card` decorators in a step with different parameters.
1208
+ Specifies environment variables to be set prior to the execution of a step.
1269
1209
 
1270
1210
  Parameters
1271
1211
  ----------
1272
- type : str, default 'default'
1273
- Card type.
1274
- id : str, optional, default None
1275
- If multiple cards are present, use this id to identify this card.
1276
- options : Dict[str, Any], default {}
1277
- Options passed to the card. The contents depend on the card type.
1278
- timeout : int, default 45
1279
- Interrupt reporting if it takes more than this many seconds.
1280
-
1281
-
1212
+ vars : Dict[str, str], default {}
1213
+ Dictionary of environment variables to set.
1282
1214
  """
1283
1215
  ...
1284
1216
 
1285
1217
  @typing.overload
1286
- def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1218
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1287
1219
  """
1288
- Specifies the flow(s) that this flow depends on.
1289
-
1290
- ```
1291
- @trigger_on_finish(flow='FooFlow')
1292
- ```
1293
- or
1294
- ```
1295
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1296
- ```
1297
- This decorator respects the @project decorator and triggers the flow
1298
- when upstream runs within the same namespace complete successfully
1299
-
1300
- Additionally, you can specify project aware upstream flow dependencies
1301
- by specifying the fully qualified project_flow_name.
1302
- ```
1303
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1304
- ```
1305
- or
1306
- ```
1307
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1308
- ```
1309
-
1310
- You can also specify just the project or project branch (other values will be
1311
- inferred from the current project or project branch):
1312
- ```
1313
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1314
- ```
1220
+ Decorator prototype for all step decorators. This function gets specialized
1221
+ and imported for all decorators types by _import_plugin_decorators().
1222
+ """
1223
+ ...
1224
+
1225
+ @typing.overload
1226
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1227
+ ...
1228
+
1229
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1230
+ """
1231
+ Decorator prototype for all step decorators. This function gets specialized
1232
+ and imported for all decorators types by _import_plugin_decorators().
1233
+ """
1234
+ ...
1235
+
1236
+ @typing.overload
1237
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1238
+ """
1239
+ Specifies the PyPI packages for the step.
1315
1240
 
1316
- Note that `branch` is typically one of:
1317
- - `prod`
1318
- - `user.bob`
1319
- - `test.my_experiment`
1320
- - `prod.staging`
1241
+ Information in this decorator will augment any
1242
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1243
+ you can use `@pypi_base` to set packages required by all
1244
+ steps and use `@pypi` to specify step-specific overrides.
1321
1245
 
1322
1246
  Parameters
1323
1247
  ----------
1324
- flow : Union[str, Dict[str, str]], optional, default None
1325
- Upstream flow dependency for this flow.
1326
- flows : List[Union[str, Dict[str, str]]], default []
1327
- Upstream flow dependencies for this flow.
1328
- options : Dict[str, Any], default {}
1329
- Backend-specific configuration for tuning eventing behavior.
1330
-
1331
-
1248
+ packages : Dict[str, str], default: {}
1249
+ Packages to use for this step. The key is the name of the package
1250
+ and the value is the version to use.
1251
+ python : str, optional, default: None
1252
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1253
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1332
1254
  """
1333
1255
  ...
1334
1256
 
1335
1257
  @typing.overload
1336
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1258
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1337
1259
  ...
1338
1260
 
1339
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1261
+ @typing.overload
1262
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1263
+ ...
1264
+
1265
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1340
1266
  """
1341
- Specifies the flow(s) that this flow depends on.
1342
-
1343
- ```
1344
- @trigger_on_finish(flow='FooFlow')
1345
- ```
1346
- or
1347
- ```
1348
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1349
- ```
1350
- This decorator respects the @project decorator and triggers the flow
1351
- when upstream runs within the same namespace complete successfully
1352
-
1353
- Additionally, you can specify project aware upstream flow dependencies
1354
- by specifying the fully qualified project_flow_name.
1355
- ```
1356
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1357
- ```
1358
- or
1359
- ```
1360
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1361
- ```
1267
+ Specifies the PyPI packages for the step.
1362
1268
 
1363
- You can also specify just the project or project branch (other values will be
1364
- inferred from the current project or project branch):
1365
- ```
1366
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1367
- ```
1269
+ Information in this decorator will augment any
1270
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1271
+ you can use `@pypi_base` to set packages required by all
1272
+ steps and use `@pypi` to specify step-specific overrides.
1368
1273
 
1369
- Note that `branch` is typically one of:
1370
- - `prod`
1371
- - `user.bob`
1372
- - `test.my_experiment`
1373
- - `prod.staging`
1274
+ Parameters
1275
+ ----------
1276
+ packages : Dict[str, str], default: {}
1277
+ Packages to use for this step. The key is the name of the package
1278
+ and the value is the version to use.
1279
+ python : str, optional, default: None
1280
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1281
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1282
+ """
1283
+ ...
1284
+
1285
+ @typing.overload
1286
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1287
+ """
1288
+ Specifies the times when the flow should be run when running on a
1289
+ production scheduler.
1374
1290
 
1375
1291
  Parameters
1376
1292
  ----------
1377
- flow : Union[str, Dict[str, str]], optional, default None
1378
- Upstream flow dependency for this flow.
1379
- flows : List[Union[str, Dict[str, str]]], default []
1380
- Upstream flow dependencies for this flow.
1381
- options : Dict[str, Any], default {}
1382
- Backend-specific configuration for tuning eventing behavior.
1293
+ hourly : bool, default False
1294
+ Run the workflow hourly.
1295
+ daily : bool, default True
1296
+ Run the workflow daily.
1297
+ weekly : bool, default False
1298
+ Run the workflow weekly.
1299
+ cron : str, optional, default None
1300
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1301
+ specified by this expression.
1302
+ timezone : str, optional, default None
1303
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1304
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1305
+ """
1306
+ ...
1307
+
1308
+ @typing.overload
1309
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1310
+ ...
1311
+
1312
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1313
+ """
1314
+ Specifies the times when the flow should be run when running on a
1315
+ production scheduler.
1383
1316
 
1317
+ Parameters
1318
+ ----------
1319
+ hourly : bool, default False
1320
+ Run the workflow hourly.
1321
+ daily : bool, default True
1322
+ Run the workflow daily.
1323
+ weekly : bool, default False
1324
+ Run the workflow weekly.
1325
+ cron : str, optional, default None
1326
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1327
+ specified by this expression.
1328
+ timezone : str, optional, default None
1329
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1330
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1331
+ """
1332
+ ...
1333
+
1334
+ @typing.overload
1335
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1336
+ """
1337
+ Specifies the PyPI packages for all steps of the flow.
1384
1338
 
1339
+ Use `@pypi_base` to set common packages required by all
1340
+ steps and use `@pypi` to specify step-specific overrides.
1341
+ Parameters
1342
+ ----------
1343
+ packages : Dict[str, str], default: {}
1344
+ Packages to use for this flow. The key is the name of the package
1345
+ and the value is the version to use.
1346
+ python : str, optional, default: None
1347
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1348
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1385
1349
  """
1386
1350
  ...
1387
1351
 
1388
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1352
+ @typing.overload
1353
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1354
+ ...
1355
+
1356
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1389
1357
  """
1390
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1391
-
1392
- User code call
1393
- -----------
1394
- @nim(
1395
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1396
- backend='managed'
1397
- )
1398
-
1399
- Valid backend options
1400
- ---------------------
1401
- - 'managed': Outerbounds selects a compute provider based on the model.
1402
- - 🚧 'dataplane': Run in your account.
1403
-
1404
- Valid model options
1405
- ----------------
1406
- - 'meta/llama3-8b-instruct': 8B parameter model
1407
- - 'meta/llama3-70b-instruct': 70B parameter model
1408
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1358
+ Specifies the PyPI packages for all steps of the flow.
1409
1359
 
1360
+ Use `@pypi_base` to set common packages required by all
1361
+ steps and use `@pypi` to specify step-specific overrides.
1410
1362
  Parameters
1411
1363
  ----------
1412
- models: list[NIM]
1413
- List of NIM containers running models in sidecars.
1414
- backend: str
1415
- Compute provider to run the NIM container.
1364
+ packages : Dict[str, str], default: {}
1365
+ Packages to use for this flow. The key is the name of the package
1366
+ and the value is the version to use.
1367
+ python : str, optional, default: None
1368
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1369
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1416
1370
  """
1417
1371
  ...
1418
1372
 
@@ -1458,6 +1412,37 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1458
1412
  """
1459
1413
  ...
1460
1414
 
1415
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1416
+ """
1417
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1418
+
1419
+ User code call
1420
+ -----------
1421
+ @nim(
1422
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1423
+ backend='managed'
1424
+ )
1425
+
1426
+ Valid backend options
1427
+ ---------------------
1428
+ - 'managed': Outerbounds selects a compute provider based on the model.
1429
+ - 🚧 'dataplane': Run in your account.
1430
+
1431
+ Valid model options
1432
+ ----------------
1433
+ - 'meta/llama3-8b-instruct': 8B parameter model
1434
+ - 'meta/llama3-70b-instruct': 70B parameter model
1435
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1436
+
1437
+ Parameters
1438
+ ----------
1439
+ models: list[NIM]
1440
+ List of NIM containers running models in sidecars.
1441
+ backend: str
1442
+ Compute provider to run the NIM container.
1443
+ """
1444
+ ...
1445
+
1461
1446
  @typing.overload
1462
1447
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1463
1448
  """
@@ -1507,6 +1492,24 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
1507
1492
  """
1508
1493
  ...
1509
1494
 
1495
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1496
+ """
1497
+ Specifies what flows belong to the same project.
1498
+
1499
+ A project-specific namespace is created for all flows that
1500
+ use the same `@project(name)`.
1501
+
1502
+ Parameters
1503
+ ----------
1504
+ name : str
1505
+ Project name. Make sure that the name is unique amongst all
1506
+ projects that use the same production scheduler. The name may
1507
+ contain only lowercase alphanumeric characters and underscores.
1508
+
1509
+
1510
+ """
1511
+ ...
1512
+
1510
1513
  @typing.overload
1511
1514
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1512
1515
  """
@@ -1602,73 +1605,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1602
1605
  """
1603
1606
  ...
1604
1607
 
1605
- @typing.overload
1606
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1607
- """
1608
- Specifies the times when the flow should be run when running on a
1609
- production scheduler.
1610
-
1611
- Parameters
1612
- ----------
1613
- hourly : bool, default False
1614
- Run the workflow hourly.
1615
- daily : bool, default True
1616
- Run the workflow daily.
1617
- weekly : bool, default False
1618
- Run the workflow weekly.
1619
- cron : str, optional, default None
1620
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1621
- specified by this expression.
1622
- timezone : str, optional, default None
1623
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1624
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1625
- """
1626
- ...
1627
-
1628
- @typing.overload
1629
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1630
- ...
1631
-
1632
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1633
- """
1634
- Specifies the times when the flow should be run when running on a
1635
- production scheduler.
1636
-
1637
- Parameters
1638
- ----------
1639
- hourly : bool, default False
1640
- Run the workflow hourly.
1641
- daily : bool, default True
1642
- Run the workflow daily.
1643
- weekly : bool, default False
1644
- Run the workflow weekly.
1645
- cron : str, optional, default None
1646
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1647
- specified by this expression.
1648
- timezone : str, optional, default None
1649
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1650
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1651
- """
1652
- ...
1653
-
1654
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1655
- """
1656
- Specifies what flows belong to the same project.
1657
-
1658
- A project-specific namespace is created for all flows that
1659
- use the same `@project(name)`.
1660
-
1661
- Parameters
1662
- ----------
1663
- name : str
1664
- Project name. Make sure that the name is unique amongst all
1665
- projects that use the same production scheduler. The name may
1666
- contain only lowercase alphanumeric characters and underscores.
1667
-
1668
-
1669
- """
1670
- ...
1671
-
1672
1608
  def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1673
1609
  """
1674
1610
  The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
@@ -1712,41 +1648,105 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1712
1648
  ...
1713
1649
 
1714
1650
  @typing.overload
1715
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1651
+ def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1716
1652
  """
1717
- Specifies the PyPI packages for all steps of the flow.
1653
+ Specifies the flow(s) that this flow depends on.
1654
+
1655
+ ```
1656
+ @trigger_on_finish(flow='FooFlow')
1657
+ ```
1658
+ or
1659
+ ```
1660
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1661
+ ```
1662
+ This decorator respects the @project decorator and triggers the flow
1663
+ when upstream runs within the same namespace complete successfully
1664
+
1665
+ Additionally, you can specify project aware upstream flow dependencies
1666
+ by specifying the fully qualified project_flow_name.
1667
+ ```
1668
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1669
+ ```
1670
+ or
1671
+ ```
1672
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1673
+ ```
1674
+
1675
+ You can also specify just the project or project branch (other values will be
1676
+ inferred from the current project or project branch):
1677
+ ```
1678
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1679
+ ```
1680
+
1681
+ Note that `branch` is typically one of:
1682
+ - `prod`
1683
+ - `user.bob`
1684
+ - `test.my_experiment`
1685
+ - `prod.staging`
1718
1686
 
1719
- Use `@pypi_base` to set common packages required by all
1720
- steps and use `@pypi` to specify step-specific overrides.
1721
1687
  Parameters
1722
1688
  ----------
1723
- packages : Dict[str, str], default: {}
1724
- Packages to use for this flow. The key is the name of the package
1725
- and the value is the version to use.
1726
- python : str, optional, default: None
1727
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1728
- that the version used will correspond to the version of the Python interpreter used to start the run.
1689
+ flow : Union[str, Dict[str, str]], optional, default None
1690
+ Upstream flow dependency for this flow.
1691
+ flows : List[Union[str, Dict[str, str]]], default []
1692
+ Upstream flow dependencies for this flow.
1693
+ options : Dict[str, Any], default {}
1694
+ Backend-specific configuration for tuning eventing behavior.
1695
+
1696
+
1729
1697
  """
1730
1698
  ...
1731
1699
 
1732
1700
  @typing.overload
1733
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1701
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1734
1702
  ...
1735
1703
 
1736
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1704
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1737
1705
  """
1738
- Specifies the PyPI packages for all steps of the flow.
1706
+ Specifies the flow(s) that this flow depends on.
1707
+
1708
+ ```
1709
+ @trigger_on_finish(flow='FooFlow')
1710
+ ```
1711
+ or
1712
+ ```
1713
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1714
+ ```
1715
+ This decorator respects the @project decorator and triggers the flow
1716
+ when upstream runs within the same namespace complete successfully
1717
+
1718
+ Additionally, you can specify project aware upstream flow dependencies
1719
+ by specifying the fully qualified project_flow_name.
1720
+ ```
1721
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1722
+ ```
1723
+ or
1724
+ ```
1725
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1726
+ ```
1727
+
1728
+ You can also specify just the project or project branch (other values will be
1729
+ inferred from the current project or project branch):
1730
+ ```
1731
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1732
+ ```
1733
+
1734
+ Note that `branch` is typically one of:
1735
+ - `prod`
1736
+ - `user.bob`
1737
+ - `test.my_experiment`
1738
+ - `prod.staging`
1739
1739
 
1740
- Use `@pypi_base` to set common packages required by all
1741
- steps and use `@pypi` to specify step-specific overrides.
1742
1740
  Parameters
1743
1741
  ----------
1744
- packages : Dict[str, str], default: {}
1745
- Packages to use for this flow. The key is the name of the package
1746
- and the value is the version to use.
1747
- python : str, optional, default: None
1748
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1749
- that the version used will correspond to the version of the Python interpreter used to start the run.
1742
+ flow : Union[str, Dict[str, str]], optional, default None
1743
+ Upstream flow dependency for this flow.
1744
+ flows : List[Union[str, Dict[str, str]]], default []
1745
+ Upstream flow dependencies for this flow.
1746
+ options : Dict[str, Any], default {}
1747
+ Backend-specific configuration for tuning eventing behavior.
1748
+
1749
+
1750
1750
  """
1751
1751
  ...
1752
1752