ob-metaflow-stubs 5.1__py2.py3-none-any.whl → 5.2__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. metaflow-stubs/__init__.pyi +493 -493
  2. metaflow-stubs/cards.pyi +4 -4
  3. metaflow-stubs/cli.pyi +3 -3
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/metadata/metadata.pyi +2 -2
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +22 -22
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +3 -3
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
  26. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  27. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  28. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  29. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
  32. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
  33. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  34. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  35. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  38. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  42. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  43. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  44. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  45. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  50. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  51. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  52. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  54. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  55. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  56. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  57. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  58. metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
  59. metaflow-stubs/plugins/cards/card_client.pyi +4 -4
  60. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  61. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  62. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  63. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  67. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  71. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  72. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  73. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  74. metaflow-stubs/plugins/datatools/__init__.pyi +4 -4
  75. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  76. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  77. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  78. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  79. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  80. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  81. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  82. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  83. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  84. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  85. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  86. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  88. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  89. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  90. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  92. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  93. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
  94. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
  95. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
  97. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  98. metaflow-stubs/plugins/logs_cli.pyi +2 -2
  99. metaflow-stubs/plugins/package_cli.pyi +2 -2
  100. metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
  101. metaflow-stubs/plugins/perimeters.pyi +2 -2
  102. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  103. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  105. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  106. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  107. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  109. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  110. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  111. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  112. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
  113. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  114. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  115. metaflow-stubs/plugins/tag_cli.pyi +5 -5
  116. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  117. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  118. metaflow-stubs/procpoll.pyi +2 -2
  119. metaflow-stubs/profilers/__init__.pyi +2 -2
  120. metaflow-stubs/pylint_wrapper.pyi +2 -2
  121. metaflow-stubs/runner/__init__.pyi +2 -2
  122. metaflow-stubs/runner/deployer.pyi +3 -3
  123. metaflow-stubs/runner/metaflow_runner.pyi +5 -5
  124. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  125. metaflow-stubs/runner/nbrun.pyi +2 -2
  126. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  127. metaflow-stubs/runner/utils.pyi +2 -2
  128. metaflow-stubs/system/__init__.pyi +3 -3
  129. metaflow-stubs/system/system_logger.pyi +2 -2
  130. metaflow-stubs/system/system_monitor.pyi +3 -3
  131. metaflow-stubs/tagging_util.pyi +2 -2
  132. metaflow-stubs/tuple_util.pyi +2 -2
  133. {ob_metaflow_stubs-5.1.dist-info → ob_metaflow_stubs-5.2.dist-info}/METADATA +1 -1
  134. ob_metaflow_stubs-5.2.dist-info/RECORD +137 -0
  135. ob_metaflow_stubs-5.1.dist-info/RECORD +0 -137
  136. {ob_metaflow_stubs-5.1.dist-info → ob_metaflow_stubs-5.2.dist-info}/WHEEL +0 -0
  137. {ob_metaflow_stubs-5.1.dist-info → ob_metaflow_stubs-5.2.dist-info}/top_level.txt +0 -0
@@ -1,25 +1,25 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.11.0+ob(v1) #
4
- # Generated on 2024-08-08T23:50:17.875353 #
3
+ # MF version: 2.12.12.0+ob(v1) #
4
+ # Generated on 2024-08-14T18:23:51.845623 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.parameters
12
- import typing
13
- import metaflow.plugins.datatools.s3.s3
14
- import metaflow.client.core
15
- import metaflow.metaflow_current
11
+ import metaflow.events
16
12
  import metaflow.flowspec
17
- import metaflow._vendor.click.types
18
13
  import io
14
+ import typing
19
15
  import datetime
20
- import metaflow.datastore.inputs
21
- import metaflow.events
22
16
  import metaflow.runner.metaflow_runner
17
+ import metaflow.client.core
18
+ import metaflow.plugins.datatools.s3.s3
19
+ import metaflow.metaflow_current
20
+ import metaflow.parameters
21
+ import metaflow.datastore.inputs
22
+ import metaflow._vendor.click.types
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
25
25
 
@@ -728,51 +728,161 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
728
728
  ...
729
729
 
730
730
  @typing.overload
731
- def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
731
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
732
  """
733
- Specifies that the step will success under all circumstances.
733
+ Specifies the Conda environment for the step.
734
734
 
735
- The decorator will create an optional artifact, specified by `var`, which
736
- contains the exception raised. You can use it to detect the presence
737
- of errors, indicating that all happy-path artifacts produced by the step
738
- are missing.
735
+ Information in this decorator will augment any
736
+ attributes set in the `@conda_base` flow-level decorator. Hence,
737
+ you can use `@conda_base` to set packages required by all
738
+ steps and use `@conda` to specify step-specific overrides.
739
739
 
740
740
  Parameters
741
741
  ----------
742
- var : str, optional, default None
743
- Name of the artifact in which to store the caught exception.
744
- If not specified, the exception is not stored.
745
- print_exception : bool, default True
746
- Determines whether or not the exception is printed to
747
- stdout when caught.
742
+ packages : Dict[str, str], default {}
743
+ Packages to use for this step. The key is the name of the package
744
+ and the value is the version to use.
745
+ libraries : Dict[str, str], default {}
746
+ Supported for backward compatibility. When used with packages, packages will take precedence.
747
+ python : str, optional, default None
748
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
749
+ that the version used will correspond to the version of the Python interpreter used to start the run.
750
+ disabled : bool, default False
751
+ If set to True, disables @conda.
748
752
  """
749
753
  ...
750
754
 
751
755
  @typing.overload
752
- def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
756
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
753
757
  ...
754
758
 
755
759
  @typing.overload
756
- def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
760
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
757
761
  ...
758
762
 
759
- def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
763
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
760
764
  """
761
- Specifies that the step will success under all circumstances.
765
+ Specifies the Conda environment for the step.
762
766
 
763
- The decorator will create an optional artifact, specified by `var`, which
764
- contains the exception raised. You can use it to detect the presence
765
- of errors, indicating that all happy-path artifacts produced by the step
766
- are missing.
767
+ Information in this decorator will augment any
768
+ attributes set in the `@conda_base` flow-level decorator. Hence,
769
+ you can use `@conda_base` to set packages required by all
770
+ steps and use `@conda` to specify step-specific overrides.
767
771
 
768
772
  Parameters
769
773
  ----------
770
- var : str, optional, default None
771
- Name of the artifact in which to store the caught exception.
772
- If not specified, the exception is not stored.
773
- print_exception : bool, default True
774
- Determines whether or not the exception is printed to
775
- stdout when caught.
774
+ packages : Dict[str, str], default {}
775
+ Packages to use for this step. The key is the name of the package
776
+ and the value is the version to use.
777
+ libraries : Dict[str, str], default {}
778
+ Supported for backward compatibility. When used with packages, packages will take precedence.
779
+ python : str, optional, default None
780
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
781
+ that the version used will correspond to the version of the Python interpreter used to start the run.
782
+ disabled : bool, default False
783
+ If set to True, disables @conda.
784
+ """
785
+ ...
786
+
787
+ @typing.overload
788
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
789
+ """
790
+ Specifies the PyPI packages for the step.
791
+
792
+ Information in this decorator will augment any
793
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
794
+ you can use `@pypi_base` to set packages required by all
795
+ steps and use `@pypi` to specify step-specific overrides.
796
+
797
+ Parameters
798
+ ----------
799
+ packages : Dict[str, str], default: {}
800
+ Packages to use for this step. The key is the name of the package
801
+ and the value is the version to use.
802
+ python : str, optional, default: None
803
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
804
+ that the version used will correspond to the version of the Python interpreter used to start the run.
805
+ """
806
+ ...
807
+
808
+ @typing.overload
809
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
810
+ ...
811
+
812
+ @typing.overload
813
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
814
+ ...
815
+
816
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
817
+ """
818
+ Specifies the PyPI packages for the step.
819
+
820
+ Information in this decorator will augment any
821
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
822
+ you can use `@pypi_base` to set packages required by all
823
+ steps and use `@pypi` to specify step-specific overrides.
824
+
825
+ Parameters
826
+ ----------
827
+ packages : Dict[str, str], default: {}
828
+ Packages to use for this step. The key is the name of the package
829
+ and the value is the version to use.
830
+ python : str, optional, default: None
831
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
832
+ that the version used will correspond to the version of the Python interpreter used to start the run.
833
+ """
834
+ ...
835
+
836
+ @typing.overload
837
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
838
+ """
839
+ Specifies the number of times the task corresponding
840
+ to a step needs to be retried.
841
+
842
+ This decorator is useful for handling transient errors, such as networking issues.
843
+ If your task contains operations that can't be retried safely, e.g. database updates,
844
+ it is advisable to annotate it with `@retry(times=0)`.
845
+
846
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
847
+ decorator will execute a no-op task after all retries have been exhausted,
848
+ ensuring that the flow execution can continue.
849
+
850
+ Parameters
851
+ ----------
852
+ times : int, default 3
853
+ Number of times to retry this task.
854
+ minutes_between_retries : int, default 2
855
+ Number of minutes between retries.
856
+ """
857
+ ...
858
+
859
+ @typing.overload
860
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
861
+ ...
862
+
863
+ @typing.overload
864
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
865
+ ...
866
+
867
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
868
+ """
869
+ Specifies the number of times the task corresponding
870
+ to a step needs to be retried.
871
+
872
+ This decorator is useful for handling transient errors, such as networking issues.
873
+ If your task contains operations that can't be retried safely, e.g. database updates,
874
+ it is advisable to annotate it with `@retry(times=0)`.
875
+
876
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
877
+ decorator will execute a no-op task after all retries have been exhausted,
878
+ ensuring that the flow execution can continue.
879
+
880
+ Parameters
881
+ ----------
882
+ times : int, default 3
883
+ Number of times to retry this task.
884
+ minutes_between_retries : int, default 2
885
+ Number of minutes between retries.
776
886
  """
777
887
  ...
778
888
 
@@ -854,164 +964,57 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
854
964
  ...
855
965
 
856
966
  @typing.overload
857
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
967
+ def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
858
968
  """
859
- Specifies a timeout for your step.
860
-
861
- This decorator is useful if this step may hang indefinitely.
862
-
863
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
864
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
865
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
969
+ Specifies that the step will success under all circumstances.
866
970
 
867
- Note that all the values specified in parameters are added together so if you specify
868
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
971
+ The decorator will create an optional artifact, specified by `var`, which
972
+ contains the exception raised. You can use it to detect the presence
973
+ of errors, indicating that all happy-path artifacts produced by the step
974
+ are missing.
869
975
 
870
976
  Parameters
871
977
  ----------
872
- seconds : int, default 0
873
- Number of seconds to wait prior to timing out.
874
- minutes : int, default 0
875
- Number of minutes to wait prior to timing out.
876
- hours : int, default 0
877
- Number of hours to wait prior to timing out.
978
+ var : str, optional, default None
979
+ Name of the artifact in which to store the caught exception.
980
+ If not specified, the exception is not stored.
981
+ print_exception : bool, default True
982
+ Determines whether or not the exception is printed to
983
+ stdout when caught.
878
984
  """
879
985
  ...
880
986
 
881
987
  @typing.overload
882
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
988
+ def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
883
989
  ...
884
990
 
885
991
  @typing.overload
886
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
992
+ def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
887
993
  ...
888
994
 
889
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
995
+ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
890
996
  """
891
- Specifies a timeout for your step.
892
-
893
- This decorator is useful if this step may hang indefinitely.
894
-
895
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
896
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
897
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
997
+ Specifies that the step will success under all circumstances.
898
998
 
899
- Note that all the values specified in parameters are added together so if you specify
900
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
999
+ The decorator will create an optional artifact, specified by `var`, which
1000
+ contains the exception raised. You can use it to detect the presence
1001
+ of errors, indicating that all happy-path artifacts produced by the step
1002
+ are missing.
901
1003
 
902
1004
  Parameters
903
1005
  ----------
904
- seconds : int, default 0
905
- Number of seconds to wait prior to timing out.
906
- minutes : int, default 0
907
- Number of minutes to wait prior to timing out.
908
- hours : int, default 0
909
- Number of hours to wait prior to timing out.
1006
+ var : str, optional, default None
1007
+ Name of the artifact in which to store the caught exception.
1008
+ If not specified, the exception is not stored.
1009
+ print_exception : bool, default True
1010
+ Determines whether or not the exception is printed to
1011
+ stdout when caught.
910
1012
  """
911
1013
  ...
912
1014
 
913
- @typing.overload
914
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1015
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
915
1016
  """
916
- Specifies environment variables to be set prior to the execution of a step.
917
-
918
- Parameters
919
- ----------
920
- vars : Dict[str, str], default {}
921
- Dictionary of environment variables to set.
922
- """
923
- ...
924
-
925
- @typing.overload
926
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
927
- ...
928
-
929
- @typing.overload
930
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
931
- ...
932
-
933
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
934
- """
935
- Specifies environment variables to be set prior to the execution of a step.
936
-
937
- Parameters
938
- ----------
939
- vars : Dict[str, str], default {}
940
- Dictionary of environment variables to set.
941
- """
942
- ...
943
-
944
- @typing.overload
945
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
946
- """
947
- Specifies the PyPI packages for the step.
948
-
949
- Information in this decorator will augment any
950
- attributes set in the `@pyi_base` flow-level decorator. Hence,
951
- you can use `@pypi_base` to set packages required by all
952
- steps and use `@pypi` to specify step-specific overrides.
953
-
954
- Parameters
955
- ----------
956
- packages : Dict[str, str], default: {}
957
- Packages to use for this step. The key is the name of the package
958
- and the value is the version to use.
959
- python : str, optional, default: None
960
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
961
- that the version used will correspond to the version of the Python interpreter used to start the run.
962
- """
963
- ...
964
-
965
- @typing.overload
966
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
967
- ...
968
-
969
- @typing.overload
970
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
971
- ...
972
-
973
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
974
- """
975
- Specifies the PyPI packages for the step.
976
-
977
- Information in this decorator will augment any
978
- attributes set in the `@pyi_base` flow-level decorator. Hence,
979
- you can use `@pypi_base` to set packages required by all
980
- steps and use `@pypi` to specify step-specific overrides.
981
-
982
- Parameters
983
- ----------
984
- packages : Dict[str, str], default: {}
985
- Packages to use for this step. The key is the name of the package
986
- and the value is the version to use.
987
- python : str, optional, default: None
988
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
989
- that the version used will correspond to the version of the Python interpreter used to start the run.
990
- """
991
- ...
992
-
993
- @typing.overload
994
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
995
- """
996
- Decorator prototype for all step decorators. This function gets specialized
997
- and imported for all decorators types by _import_plugin_decorators().
998
- """
999
- ...
1000
-
1001
- @typing.overload
1002
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1003
- ...
1004
-
1005
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1006
- """
1007
- Decorator prototype for all step decorators. This function gets specialized
1008
- and imported for all decorators types by _import_plugin_decorators().
1009
- """
1010
- ...
1011
-
1012
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1013
- """
1014
- Specifies that this step should execute on Kubernetes.
1017
+ Specifies that this step should execute on Kubernetes.
1015
1018
 
1016
1019
  Parameters
1017
1020
  ----------
@@ -1068,116 +1071,6 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
1068
1071
  """
1069
1072
  ...
1070
1073
 
1071
- @typing.overload
1072
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1073
- """
1074
- Specifies the number of times the task corresponding
1075
- to a step needs to be retried.
1076
-
1077
- This decorator is useful for handling transient errors, such as networking issues.
1078
- If your task contains operations that can't be retried safely, e.g. database updates,
1079
- it is advisable to annotate it with `@retry(times=0)`.
1080
-
1081
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1082
- decorator will execute a no-op task after all retries have been exhausted,
1083
- ensuring that the flow execution can continue.
1084
-
1085
- Parameters
1086
- ----------
1087
- times : int, default 3
1088
- Number of times to retry this task.
1089
- minutes_between_retries : int, default 2
1090
- Number of minutes between retries.
1091
- """
1092
- ...
1093
-
1094
- @typing.overload
1095
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1096
- ...
1097
-
1098
- @typing.overload
1099
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1100
- ...
1101
-
1102
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1103
- """
1104
- Specifies the number of times the task corresponding
1105
- to a step needs to be retried.
1106
-
1107
- This decorator is useful for handling transient errors, such as networking issues.
1108
- If your task contains operations that can't be retried safely, e.g. database updates,
1109
- it is advisable to annotate it with `@retry(times=0)`.
1110
-
1111
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1112
- decorator will execute a no-op task after all retries have been exhausted,
1113
- ensuring that the flow execution can continue.
1114
-
1115
- Parameters
1116
- ----------
1117
- times : int, default 3
1118
- Number of times to retry this task.
1119
- minutes_between_retries : int, default 2
1120
- Number of minutes between retries.
1121
- """
1122
- ...
1123
-
1124
- @typing.overload
1125
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1126
- """
1127
- Specifies the Conda environment for the step.
1128
-
1129
- Information in this decorator will augment any
1130
- attributes set in the `@conda_base` flow-level decorator. Hence,
1131
- you can use `@conda_base` to set packages required by all
1132
- steps and use `@conda` to specify step-specific overrides.
1133
-
1134
- Parameters
1135
- ----------
1136
- packages : Dict[str, str], default {}
1137
- Packages to use for this step. The key is the name of the package
1138
- and the value is the version to use.
1139
- libraries : Dict[str, str], default {}
1140
- Supported for backward compatibility. When used with packages, packages will take precedence.
1141
- python : str, optional, default None
1142
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1143
- that the version used will correspond to the version of the Python interpreter used to start the run.
1144
- disabled : bool, default False
1145
- If set to True, disables @conda.
1146
- """
1147
- ...
1148
-
1149
- @typing.overload
1150
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1151
- ...
1152
-
1153
- @typing.overload
1154
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1155
- ...
1156
-
1157
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1158
- """
1159
- Specifies the Conda environment for the step.
1160
-
1161
- Information in this decorator will augment any
1162
- attributes set in the `@conda_base` flow-level decorator. Hence,
1163
- you can use `@conda_base` to set packages required by all
1164
- steps and use `@conda` to specify step-specific overrides.
1165
-
1166
- Parameters
1167
- ----------
1168
- packages : Dict[str, str], default {}
1169
- Packages to use for this step. The key is the name of the package
1170
- and the value is the version to use.
1171
- libraries : Dict[str, str], default {}
1172
- Supported for backward compatibility. When used with packages, packages will take precedence.
1173
- python : str, optional, default None
1174
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1175
- that the version used will correspond to the version of the Python interpreter used to start the run.
1176
- disabled : bool, default False
1177
- If set to True, disables @conda.
1178
- """
1179
- ...
1180
-
1181
1074
  @typing.overload
1182
1075
  def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1183
1076
  """
@@ -1229,6 +1122,25 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
1229
1122
  """
1230
1123
  ...
1231
1124
 
1125
+ @typing.overload
1126
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1127
+ """
1128
+ Decorator prototype for all step decorators. This function gets specialized
1129
+ and imported for all decorators types by _import_plugin_decorators().
1130
+ """
1131
+ ...
1132
+
1133
+ @typing.overload
1134
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1135
+ ...
1136
+
1137
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1138
+ """
1139
+ Decorator prototype for all step decorators. This function gets specialized
1140
+ and imported for all decorators types by _import_plugin_decorators().
1141
+ """
1142
+ ...
1143
+
1232
1144
  @typing.overload
1233
1145
  def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1234
1146
  """
@@ -1263,160 +1175,132 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
1263
1175
  ...
1264
1176
 
1265
1177
  @typing.overload
1266
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1178
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1267
1179
  """
1268
- Specifies the PyPI packages for all steps of the flow.
1180
+ Specifies environment variables to be set prior to the execution of a step.
1269
1181
 
1270
- Use `@pypi_base` to set common packages required by all
1271
- steps and use `@pypi` to specify step-specific overrides.
1272
1182
  Parameters
1273
1183
  ----------
1274
- packages : Dict[str, str], default: {}
1275
- Packages to use for this flow. The key is the name of the package
1276
- and the value is the version to use.
1277
- python : str, optional, default: None
1278
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1279
- that the version used will correspond to the version of the Python interpreter used to start the run.
1184
+ vars : Dict[str, str], default {}
1185
+ Dictionary of environment variables to set.
1280
1186
  """
1281
1187
  ...
1282
1188
 
1283
1189
  @typing.overload
1284
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1190
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1285
1191
  ...
1286
1192
 
1287
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1193
+ @typing.overload
1194
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1195
+ ...
1196
+
1197
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1288
1198
  """
1289
- Specifies the PyPI packages for all steps of the flow.
1199
+ Specifies environment variables to be set prior to the execution of a step.
1290
1200
 
1291
- Use `@pypi_base` to set common packages required by all
1292
- steps and use `@pypi` to specify step-specific overrides.
1293
1201
  Parameters
1294
1202
  ----------
1295
- packages : Dict[str, str], default: {}
1296
- Packages to use for this flow. The key is the name of the package
1297
- and the value is the version to use.
1298
- python : str, optional, default: None
1299
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1300
- that the version used will correspond to the version of the Python interpreter used to start the run.
1203
+ vars : Dict[str, str], default {}
1204
+ Dictionary of environment variables to set.
1301
1205
  """
1302
1206
  ...
1303
1207
 
1304
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1208
+ @typing.overload
1209
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1305
1210
  """
1306
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1211
+ Specifies a timeout for your step.
1307
1212
 
1308
- User code call
1309
- -----------
1310
- @nim(
1311
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1312
- backend='managed'
1313
- )
1213
+ This decorator is useful if this step may hang indefinitely.
1314
1214
 
1315
- Valid backend options
1316
- ---------------------
1317
- - 'managed': Outerbounds selects a compute provider based on the model.
1318
- - 🚧 'dataplane': Run in your account.
1215
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1216
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1217
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1319
1218
 
1320
- Valid model options
1321
- ----------------
1322
- - 'meta/llama3-8b-instruct': 8B parameter model
1323
- - 'meta/llama3-70b-instruct': 70B parameter model
1324
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1219
+ Note that all the values specified in parameters are added together so if you specify
1220
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1325
1221
 
1326
1222
  Parameters
1327
1223
  ----------
1328
- models: list[NIM]
1329
- List of NIM containers running models in sidecars.
1330
- backend: str
1331
- Compute provider to run the NIM container.
1224
+ seconds : int, default 0
1225
+ Number of seconds to wait prior to timing out.
1226
+ minutes : int, default 0
1227
+ Number of minutes to wait prior to timing out.
1228
+ hours : int, default 0
1229
+ Number of hours to wait prior to timing out.
1332
1230
  """
1333
1231
  ...
1334
1232
 
1335
- def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1233
+ @typing.overload
1234
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1235
+ ...
1236
+
1237
+ @typing.overload
1238
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1239
+ ...
1240
+
1241
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1336
1242
  """
1337
- The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1338
- before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1339
- and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1340
- added as a flow decorators. Adding more than one decorator will ensure that `start` step
1341
- starts only after all sensors finish.
1243
+ Specifies a timeout for your step.
1244
+
1245
+ This decorator is useful if this step may hang indefinitely.
1246
+
1247
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1248
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1249
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1250
+
1251
+ Note that all the values specified in parameters are added together so if you specify
1252
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1342
1253
 
1343
1254
  Parameters
1344
1255
  ----------
1345
- timeout : int
1346
- Time, in seconds before the task times out and fails. (Default: 3600)
1347
- poke_interval : int
1348
- Time in seconds that the job should wait in between each try. (Default: 60)
1349
- mode : str
1350
- How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1351
- exponential_backoff : bool
1352
- allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1353
- pool : str
1354
- the slot pool this task should run in,
1355
- slot pools are a way to limit concurrency for certain tasks. (Default:None)
1356
- soft_fail : bool
1357
- Set to true to mark the task as SKIPPED on failure. (Default: False)
1358
- name : str
1359
- Name of the sensor on Airflow
1360
- description : str
1361
- Description of sensor in the Airflow UI
1362
- bucket_key : Union[str, List[str]]
1363
- The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1364
- When it's specified as a full s3:// url, please leave `bucket_name` as None
1365
- bucket_name : str
1366
- Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1367
- When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1368
- wildcard_match : bool
1369
- whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1370
- aws_conn_id : str
1371
- a reference to the s3 connection on Airflow. (Default: None)
1372
- verify : bool
1373
- Whether or not to verify SSL certificates for S3 connection. (Default: None)
1256
+ seconds : int, default 0
1257
+ Number of seconds to wait prior to timing out.
1258
+ minutes : int, default 0
1259
+ Number of minutes to wait prior to timing out.
1260
+ hours : int, default 0
1261
+ Number of hours to wait prior to timing out.
1374
1262
  """
1375
1263
  ...
1376
1264
 
1377
1265
  @typing.overload
1378
- def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1266
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1379
1267
  """
1380
- Specifies the flow(s) that this flow depends on.
1268
+ Specifies the event(s) that this flow depends on.
1381
1269
 
1382
1270
  ```
1383
- @trigger_on_finish(flow='FooFlow')
1271
+ @trigger(event='foo')
1384
1272
  ```
1385
1273
  or
1386
1274
  ```
1387
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1275
+ @trigger(events=['foo', 'bar'])
1388
1276
  ```
1389
- This decorator respects the @project decorator and triggers the flow
1390
- when upstream runs within the same namespace complete successfully
1391
1277
 
1392
- Additionally, you can specify project aware upstream flow dependencies
1393
- by specifying the fully qualified project_flow_name.
1278
+ Additionally, you can specify the parameter mappings
1279
+ to map event payload to Metaflow parameters for the flow.
1394
1280
  ```
1395
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1281
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1396
1282
  ```
1397
1283
  or
1398
1284
  ```
1399
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1285
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1286
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1400
1287
  ```
1401
1288
 
1402
- You can also specify just the project or project branch (other values will be
1403
- inferred from the current project or project branch):
1289
+ 'parameters' can also be a list of strings and tuples like so:
1404
1290
  ```
1405
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1291
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1292
+ ```
1293
+ This is equivalent to:
1294
+ ```
1295
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1406
1296
  ```
1407
-
1408
- Note that `branch` is typically one of:
1409
- - `prod`
1410
- - `user.bob`
1411
- - `test.my_experiment`
1412
- - `prod.staging`
1413
1297
 
1414
1298
  Parameters
1415
1299
  ----------
1416
- flow : Union[str, Dict[str, str]], optional, default None
1417
- Upstream flow dependency for this flow.
1418
- flows : List[Union[str, Dict[str, str]]], default []
1419
- Upstream flow dependencies for this flow.
1300
+ event : Union[str, Dict[str, Any]], optional, default None
1301
+ Event dependency for this flow.
1302
+ events : List[Union[str, Dict[str, Any]]], default []
1303
+ Events dependency for this flow.
1420
1304
  options : Dict[str, Any], default {}
1421
1305
  Backend-specific configuration for tuning eventing behavior.
1422
1306
 
@@ -1425,51 +1309,47 @@ def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] =
1425
1309
  ...
1426
1310
 
1427
1311
  @typing.overload
1428
- def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1312
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1429
1313
  ...
1430
1314
 
1431
- def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1315
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1432
1316
  """
1433
- Specifies the flow(s) that this flow depends on.
1317
+ Specifies the event(s) that this flow depends on.
1434
1318
 
1435
1319
  ```
1436
- @trigger_on_finish(flow='FooFlow')
1320
+ @trigger(event='foo')
1437
1321
  ```
1438
1322
  or
1439
1323
  ```
1440
- @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1324
+ @trigger(events=['foo', 'bar'])
1441
1325
  ```
1442
- This decorator respects the @project decorator and triggers the flow
1443
- when upstream runs within the same namespace complete successfully
1444
1326
 
1445
- Additionally, you can specify project aware upstream flow dependencies
1446
- by specifying the fully qualified project_flow_name.
1327
+ Additionally, you can specify the parameter mappings
1328
+ to map event payload to Metaflow parameters for the flow.
1447
1329
  ```
1448
- @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1330
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1449
1331
  ```
1450
1332
  or
1451
1333
  ```
1452
- @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1334
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1335
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1453
1336
  ```
1454
1337
 
1455
- You can also specify just the project or project branch (other values will be
1456
- inferred from the current project or project branch):
1338
+ 'parameters' can also be a list of strings and tuples like so:
1457
1339
  ```
1458
- @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1340
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1341
+ ```
1342
+ This is equivalent to:
1343
+ ```
1344
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1459
1345
  ```
1460
-
1461
- Note that `branch` is typically one of:
1462
- - `prod`
1463
- - `user.bob`
1464
- - `test.my_experiment`
1465
- - `prod.staging`
1466
1346
 
1467
1347
  Parameters
1468
1348
  ----------
1469
- flow : Union[str, Dict[str, str]], optional, default None
1470
- Upstream flow dependency for this flow.
1471
- flows : List[Union[str, Dict[str, str]]], default []
1472
- Upstream flow dependencies for this flow.
1349
+ event : Union[str, Dict[str, Any]], optional, default None
1350
+ Event dependency for this flow.
1351
+ events : List[Union[str, Dict[str, Any]]], default []
1352
+ Events dependency for this flow.
1473
1353
  options : Dict[str, Any], default {}
1474
1354
  Backend-specific configuration for tuning eventing behavior.
1475
1355
 
@@ -1477,6 +1357,45 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1477
1357
  """
1478
1358
  ...
1479
1359
 
1360
+ @typing.overload
1361
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1362
+ """
1363
+ Specifies the PyPI packages for all steps of the flow.
1364
+
1365
+ Use `@pypi_base` to set common packages required by all
1366
+ steps and use `@pypi` to specify step-specific overrides.
1367
+ Parameters
1368
+ ----------
1369
+ packages : Dict[str, str], default: {}
1370
+ Packages to use for this flow. The key is the name of the package
1371
+ and the value is the version to use.
1372
+ python : str, optional, default: None
1373
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1374
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1375
+ """
1376
+ ...
1377
+
1378
+ @typing.overload
1379
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1380
+ ...
1381
+
1382
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1383
+ """
1384
+ Specifies the PyPI packages for all steps of the flow.
1385
+
1386
+ Use `@pypi_base` to set common packages required by all
1387
+ steps and use `@pypi` to specify step-specific overrides.
1388
+ Parameters
1389
+ ----------
1390
+ packages : Dict[str, str], default: {}
1391
+ Packages to use for this flow. The key is the name of the package
1392
+ and the value is the version to use.
1393
+ python : str, optional, default: None
1394
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1395
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1396
+ """
1397
+ ...
1398
+
1480
1399
  def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1481
1400
  """
1482
1401
  The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
@@ -1520,44 +1439,97 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1520
1439
  ...
1521
1440
 
1522
1441
  @typing.overload
1523
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1442
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1524
1443
  """
1525
- Specifies the event(s) that this flow depends on.
1444
+ Specifies the times when the flow should be run when running on a
1445
+ production scheduler.
1446
+
1447
+ Parameters
1448
+ ----------
1449
+ hourly : bool, default False
1450
+ Run the workflow hourly.
1451
+ daily : bool, default True
1452
+ Run the workflow daily.
1453
+ weekly : bool, default False
1454
+ Run the workflow weekly.
1455
+ cron : str, optional, default None
1456
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1457
+ specified by this expression.
1458
+ timezone : str, optional, default None
1459
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1460
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1461
+ """
1462
+ ...
1463
+
1464
+ @typing.overload
1465
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1466
+ ...
1467
+
1468
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1469
+ """
1470
+ Specifies the times when the flow should be run when running on a
1471
+ production scheduler.
1472
+
1473
+ Parameters
1474
+ ----------
1475
+ hourly : bool, default False
1476
+ Run the workflow hourly.
1477
+ daily : bool, default True
1478
+ Run the workflow daily.
1479
+ weekly : bool, default False
1480
+ Run the workflow weekly.
1481
+ cron : str, optional, default None
1482
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1483
+ specified by this expression.
1484
+ timezone : str, optional, default None
1485
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1486
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1487
+ """
1488
+ ...
1489
+
1490
+ @typing.overload
1491
+ def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1492
+ """
1493
+ Specifies the flow(s) that this flow depends on.
1526
1494
 
1527
1495
  ```
1528
- @trigger(event='foo')
1496
+ @trigger_on_finish(flow='FooFlow')
1529
1497
  ```
1530
1498
  or
1531
1499
  ```
1532
- @trigger(events=['foo', 'bar'])
1500
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1533
1501
  ```
1502
+ This decorator respects the @project decorator and triggers the flow
1503
+ when upstream runs within the same namespace complete successfully
1534
1504
 
1535
- Additionally, you can specify the parameter mappings
1536
- to map event payload to Metaflow parameters for the flow.
1505
+ Additionally, you can specify project aware upstream flow dependencies
1506
+ by specifying the fully qualified project_flow_name.
1537
1507
  ```
1538
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1508
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1539
1509
  ```
1540
1510
  or
1541
1511
  ```
1542
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1543
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1512
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1544
1513
  ```
1545
1514
 
1546
- 'parameters' can also be a list of strings and tuples like so:
1547
- ```
1548
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1549
- ```
1550
- This is equivalent to:
1515
+ You can also specify just the project or project branch (other values will be
1516
+ inferred from the current project or project branch):
1551
1517
  ```
1552
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1518
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1553
1519
  ```
1554
1520
 
1521
+ Note that `branch` is typically one of:
1522
+ - `prod`
1523
+ - `user.bob`
1524
+ - `test.my_experiment`
1525
+ - `prod.staging`
1526
+
1555
1527
  Parameters
1556
1528
  ----------
1557
- event : Union[str, Dict[str, Any]], optional, default None
1558
- Event dependency for this flow.
1559
- events : List[Union[str, Dict[str, Any]]], default []
1560
- Events dependency for this flow.
1529
+ flow : Union[str, Dict[str, str]], optional, default None
1530
+ Upstream flow dependency for this flow.
1531
+ flows : List[Union[str, Dict[str, str]]], default []
1532
+ Upstream flow dependencies for this flow.
1561
1533
  options : Dict[str, Any], default {}
1562
1534
  Backend-specific configuration for tuning eventing behavior.
1563
1535
 
@@ -1566,47 +1538,51 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
1566
1538
  ...
1567
1539
 
1568
1540
  @typing.overload
1569
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1541
+ def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1570
1542
  ...
1571
1543
 
1572
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1544
+ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
1573
1545
  """
1574
- Specifies the event(s) that this flow depends on.
1546
+ Specifies the flow(s) that this flow depends on.
1575
1547
 
1576
1548
  ```
1577
- @trigger(event='foo')
1549
+ @trigger_on_finish(flow='FooFlow')
1578
1550
  ```
1579
1551
  or
1580
1552
  ```
1581
- @trigger(events=['foo', 'bar'])
1553
+ @trigger_on_finish(flows=['FooFlow', 'BarFlow'])
1582
1554
  ```
1555
+ This decorator respects the @project decorator and triggers the flow
1556
+ when upstream runs within the same namespace complete successfully
1583
1557
 
1584
- Additionally, you can specify the parameter mappings
1585
- to map event payload to Metaflow parameters for the flow.
1558
+ Additionally, you can specify project aware upstream flow dependencies
1559
+ by specifying the fully qualified project_flow_name.
1586
1560
  ```
1587
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1561
+ @trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
1588
1562
  ```
1589
1563
  or
1590
1564
  ```
1591
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1592
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1565
+ @trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
1593
1566
  ```
1594
1567
 
1595
- 'parameters' can also be a list of strings and tuples like so:
1596
- ```
1597
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1598
- ```
1599
- This is equivalent to:
1568
+ You can also specify just the project or project branch (other values will be
1569
+ inferred from the current project or project branch):
1600
1570
  ```
1601
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1571
+ @trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
1602
1572
  ```
1603
1573
 
1574
+ Note that `branch` is typically one of:
1575
+ - `prod`
1576
+ - `user.bob`
1577
+ - `test.my_experiment`
1578
+ - `prod.staging`
1579
+
1604
1580
  Parameters
1605
1581
  ----------
1606
- event : Union[str, Dict[str, Any]], optional, default None
1607
- Event dependency for this flow.
1608
- events : List[Union[str, Dict[str, Any]]], default []
1609
- Events dependency for this flow.
1582
+ flow : Union[str, Dict[str, str]], optional, default None
1583
+ Upstream flow dependency for this flow.
1584
+ flows : List[Union[str, Dict[str, str]]], default []
1585
+ Upstream flow dependencies for this flow.
1610
1586
  options : Dict[str, Any], default {}
1611
1587
  Backend-specific configuration for tuning eventing behavior.
1612
1588
 
@@ -1614,55 +1590,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1614
1590
  """
1615
1591
  ...
1616
1592
 
1617
- @typing.overload
1618
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1619
- """
1620
- Specifies the times when the flow should be run when running on a
1621
- production scheduler.
1622
-
1623
- Parameters
1624
- ----------
1625
- hourly : bool, default False
1626
- Run the workflow hourly.
1627
- daily : bool, default True
1628
- Run the workflow daily.
1629
- weekly : bool, default False
1630
- Run the workflow weekly.
1631
- cron : str, optional, default None
1632
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1633
- specified by this expression.
1634
- timezone : str, optional, default None
1635
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1636
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1637
- """
1638
- ...
1639
-
1640
- @typing.overload
1641
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1642
- ...
1643
-
1644
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1645
- """
1646
- Specifies the times when the flow should be run when running on a
1647
- production scheduler.
1648
-
1649
- Parameters
1650
- ----------
1651
- hourly : bool, default False
1652
- Run the workflow hourly.
1653
- daily : bool, default True
1654
- Run the workflow daily.
1655
- weekly : bool, default False
1656
- Run the workflow weekly.
1657
- cron : str, optional, default None
1658
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1659
- specified by this expression.
1660
- timezone : str, optional, default None
1661
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1662
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1663
- """
1664
- ...
1665
-
1666
1593
  @typing.overload
1667
1594
  def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1668
1595
  """
@@ -1730,6 +1657,79 @@ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typ
1730
1657
  """
1731
1658
  ...
1732
1659
 
1660
+ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1661
+ """
1662
+ The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
1663
+ before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
1664
+ and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
1665
+ added as a flow decorators. Adding more than one decorator will ensure that `start` step
1666
+ starts only after all sensors finish.
1667
+
1668
+ Parameters
1669
+ ----------
1670
+ timeout : int
1671
+ Time, in seconds before the task times out and fails. (Default: 3600)
1672
+ poke_interval : int
1673
+ Time in seconds that the job should wait in between each try. (Default: 60)
1674
+ mode : str
1675
+ How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
1676
+ exponential_backoff : bool
1677
+ allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
1678
+ pool : str
1679
+ the slot pool this task should run in,
1680
+ slot pools are a way to limit concurrency for certain tasks. (Default:None)
1681
+ soft_fail : bool
1682
+ Set to true to mark the task as SKIPPED on failure. (Default: False)
1683
+ name : str
1684
+ Name of the sensor on Airflow
1685
+ description : str
1686
+ Description of sensor in the Airflow UI
1687
+ bucket_key : Union[str, List[str]]
1688
+ The key(s) being waited on. Supports full s3:// style url or relative path from root level.
1689
+ When it's specified as a full s3:// url, please leave `bucket_name` as None
1690
+ bucket_name : str
1691
+ Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
1692
+ When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
1693
+ wildcard_match : bool
1694
+ whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
1695
+ aws_conn_id : str
1696
+ a reference to the s3 connection on Airflow. (Default: None)
1697
+ verify : bool
1698
+ Whether or not to verify SSL certificates for S3 connection. (Default: None)
1699
+ """
1700
+ ...
1701
+
1702
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1703
+ """
1704
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1705
+
1706
+ User code call
1707
+ -----------
1708
+ @nim(
1709
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1710
+ backend='managed'
1711
+ )
1712
+
1713
+ Valid backend options
1714
+ ---------------------
1715
+ - 'managed': Outerbounds selects a compute provider based on the model.
1716
+ - 🚧 'dataplane': Run in your account.
1717
+
1718
+ Valid model options
1719
+ ----------------
1720
+ - 'meta/llama3-8b-instruct': 8B parameter model
1721
+ - 'meta/llama3-70b-instruct': 70B parameter model
1722
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1723
+
1724
+ Parameters
1725
+ ----------
1726
+ models: list[NIM]
1727
+ List of NIM containers running models in sidecars.
1728
+ backend: str
1729
+ Compute provider to run the NIM container.
1730
+ """
1731
+ ...
1732
+
1733
1733
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1734
1734
  """
1735
1735
  Switch namespace to the one provided.