ob-metaflow-stubs 5.3__py2.py3-none-any.whl → 5.4__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. metaflow-stubs/__init__.pyi +428 -428
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +3 -3
  4. metaflow-stubs/client/__init__.pyi +4 -4
  5. metaflow-stubs/client/core.pyi +6 -6
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +2 -2
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +5 -5
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +4 -4
  13. metaflow-stubs/metadata/metadata.pyi +2 -2
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +2 -2
  16. metaflow-stubs/metaflow_current.pyi +4 -4
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +3 -3
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  23. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
  26. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +4 -4
  27. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +4 -4
  28. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  29. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  30. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  31. metaflow-stubs/plugins/argo/argo_workflows.pyi +5 -5
  32. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
  33. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +6 -6
  34. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  35. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  36. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  37. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  38. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  39. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +4 -4
  42. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  43. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  44. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  45. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  46. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  47. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
  51. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  52. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  53. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  54. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  55. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  56. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  57. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  58. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  59. metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
  60. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  61. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  62. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  63. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  64. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  65. metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
  66. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
  68. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  69. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  71. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  72. metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
  73. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  74. metaflow-stubs/plugins/catch_decorator.pyi +4 -4
  75. metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
  76. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  77. metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
  78. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  79. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  80. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  81. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  82. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  83. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  84. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  85. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/frameworks/pytorch.pyi +4 -4
  87. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  88. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  89. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  90. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  91. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  92. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  93. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  94. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
  95. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
  96. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  97. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +4 -4
  98. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  99. metaflow-stubs/plugins/logs_cli.pyi +3 -3
  100. metaflow-stubs/plugins/package_cli.pyi +2 -2
  101. metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
  102. metaflow-stubs/plugins/perimeters.pyi +2 -2
  103. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  104. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  105. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  106. metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
  107. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  108. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  109. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  110. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  111. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  112. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  113. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  114. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  115. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  116. metaflow-stubs/plugins/tag_cli.pyi +3 -3
  117. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
  118. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  119. metaflow-stubs/procpoll.pyi +2 -2
  120. metaflow-stubs/profilers/__init__.pyi +2 -2
  121. metaflow-stubs/pylint_wrapper.pyi +2 -2
  122. metaflow-stubs/runner/__init__.pyi +2 -2
  123. metaflow-stubs/runner/deployer.pyi +4 -4
  124. metaflow-stubs/runner/metaflow_runner.pyi +5 -5
  125. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  126. metaflow-stubs/runner/nbrun.pyi +2 -2
  127. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  128. metaflow-stubs/runner/utils.pyi +2 -2
  129. metaflow-stubs/system/__init__.pyi +3 -3
  130. metaflow-stubs/system/system_logger.pyi +2 -2
  131. metaflow-stubs/system/system_monitor.pyi +3 -3
  132. metaflow-stubs/tagging_util.pyi +2 -2
  133. metaflow-stubs/tuple_util.pyi +2 -2
  134. {ob_metaflow_stubs-5.3.dist-info → ob_metaflow_stubs-5.4.dist-info}/METADATA +1 -1
  135. ob_metaflow_stubs-5.4.dist-info/RECORD +138 -0
  136. ob_metaflow_stubs-5.3.dist-info/RECORD +0 -138
  137. {ob_metaflow_stubs-5.3.dist-info → ob_metaflow_stubs-5.4.dist-info}/WHEEL +0 -0
  138. {ob_metaflow_stubs-5.3.dist-info → ob_metaflow_stubs-5.4.dist-info}/top_level.txt +0 -0
@@ -1,24 +1,24 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.12.15.1+ob(v1) #
4
- # Generated on 2024-08-22T21:03:01.630360 #
3
+ # MF version: 2.12.16.1+ob(v1) #
4
+ # Generated on 2024-08-26T23:03:56.776768 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.flowspec
11
+ import io
12
+ import metaflow.client.core
13
+ import metaflow._vendor.click.types
12
14
  import metaflow.parameters
13
15
  import metaflow.plugins.datatools.s3.s3
14
- import io
15
- import datetime
16
16
  import metaflow.events
17
- import metaflow.datastore.inputs
17
+ import datetime
18
18
  import metaflow.metaflow_current
19
- import metaflow.client.core
20
19
  import metaflow.runner.metaflow_runner
21
- import metaflow._vendor.click.types
20
+ import metaflow.datastore.inputs
21
+ import metaflow.flowspec
22
22
  import typing
23
23
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
24
24
  StepFlag = typing.NewType("StepFlag", bool)
@@ -727,6 +727,37 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
727
727
  """
728
728
  ...
729
729
 
730
+ @typing.overload
731
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
+ """
733
+ Specifies environment variables to be set prior to the execution of a step.
734
+
735
+ Parameters
736
+ ----------
737
+ vars : Dict[str, str], default {}
738
+ Dictionary of environment variables to set.
739
+ """
740
+ ...
741
+
742
+ @typing.overload
743
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
744
+ ...
745
+
746
+ @typing.overload
747
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
748
+ ...
749
+
750
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
751
+ """
752
+ Specifies environment variables to be set prior to the execution of a step.
753
+
754
+ Parameters
755
+ ----------
756
+ vars : Dict[str, str], default {}
757
+ Dictionary of environment variables to set.
758
+ """
759
+ ...
760
+
730
761
  @typing.overload
731
762
  def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
732
763
  """
@@ -776,6 +807,55 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
776
807
  """
777
808
  ...
778
809
 
810
+ @typing.overload
811
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
812
+ """
813
+ Specifies the PyPI packages for the step.
814
+
815
+ Information in this decorator will augment any
816
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
817
+ you can use `@pypi_base` to set packages required by all
818
+ steps and use `@pypi` to specify step-specific overrides.
819
+
820
+ Parameters
821
+ ----------
822
+ packages : Dict[str, str], default: {}
823
+ Packages to use for this step. The key is the name of the package
824
+ and the value is the version to use.
825
+ python : str, optional, default: None
826
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
827
+ that the version used will correspond to the version of the Python interpreter used to start the run.
828
+ """
829
+ ...
830
+
831
+ @typing.overload
832
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
833
+ ...
834
+
835
+ @typing.overload
836
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
837
+ ...
838
+
839
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
840
+ """
841
+ Specifies the PyPI packages for the step.
842
+
843
+ Information in this decorator will augment any
844
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
845
+ you can use `@pypi_base` to set packages required by all
846
+ steps and use `@pypi` to specify step-specific overrides.
847
+
848
+ Parameters
849
+ ----------
850
+ packages : Dict[str, str], default: {}
851
+ Packages to use for this step. The key is the name of the package
852
+ and the value is the version to use.
853
+ python : str, optional, default: None
854
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
855
+ that the version used will correspond to the version of the Python interpreter used to start the run.
856
+ """
857
+ ...
858
+
779
859
  @typing.overload
780
860
  def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
781
861
  """
@@ -833,6 +913,149 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
833
913
  """
834
914
  ...
835
915
 
916
+ @typing.overload
917
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
918
+ """
919
+ Specifies secrets to be retrieved and injected as environment variables prior to
920
+ the execution of a step.
921
+
922
+ Parameters
923
+ ----------
924
+ sources : List[Union[str, Dict[str, Any]]], default: []
925
+ List of secret specs, defining how the secrets are to be retrieved
926
+ """
927
+ ...
928
+
929
+ @typing.overload
930
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
931
+ ...
932
+
933
+ @typing.overload
934
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
935
+ ...
936
+
937
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
938
+ """
939
+ Specifies secrets to be retrieved and injected as environment variables prior to
940
+ the execution of a step.
941
+
942
+ Parameters
943
+ ----------
944
+ sources : List[Union[str, Dict[str, Any]]], default: []
945
+ List of secret specs, defining how the secrets are to be retrieved
946
+ """
947
+ ...
948
+
949
+ @typing.overload
950
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
951
+ """
952
+ Specifies the number of times the task corresponding
953
+ to a step needs to be retried.
954
+
955
+ This decorator is useful for handling transient errors, such as networking issues.
956
+ If your task contains operations that can't be retried safely, e.g. database updates,
957
+ it is advisable to annotate it with `@retry(times=0)`.
958
+
959
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
960
+ decorator will execute a no-op task after all retries have been exhausted,
961
+ ensuring that the flow execution can continue.
962
+
963
+ Parameters
964
+ ----------
965
+ times : int, default 3
966
+ Number of times to retry this task.
967
+ minutes_between_retries : int, default 2
968
+ Number of minutes between retries.
969
+ """
970
+ ...
971
+
972
+ @typing.overload
973
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
974
+ ...
975
+
976
+ @typing.overload
977
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
978
+ ...
979
+
980
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
981
+ """
982
+ Specifies the number of times the task corresponding
983
+ to a step needs to be retried.
984
+
985
+ This decorator is useful for handling transient errors, such as networking issues.
986
+ If your task contains operations that can't be retried safely, e.g. database updates,
987
+ it is advisable to annotate it with `@retry(times=0)`.
988
+
989
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
990
+ decorator will execute a no-op task after all retries have been exhausted,
991
+ ensuring that the flow execution can continue.
992
+
993
+ Parameters
994
+ ----------
995
+ times : int, default 3
996
+ Number of times to retry this task.
997
+ minutes_between_retries : int, default 2
998
+ Number of minutes between retries.
999
+ """
1000
+ ...
1001
+
1002
+ @typing.overload
1003
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1004
+ """
1005
+ Specifies the Conda environment for the step.
1006
+
1007
+ Information in this decorator will augment any
1008
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1009
+ you can use `@conda_base` to set packages required by all
1010
+ steps and use `@conda` to specify step-specific overrides.
1011
+
1012
+ Parameters
1013
+ ----------
1014
+ packages : Dict[str, str], default {}
1015
+ Packages to use for this step. The key is the name of the package
1016
+ and the value is the version to use.
1017
+ libraries : Dict[str, str], default {}
1018
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1019
+ python : str, optional, default None
1020
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1021
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1022
+ disabled : bool, default False
1023
+ If set to True, disables @conda.
1024
+ """
1025
+ ...
1026
+
1027
+ @typing.overload
1028
+ def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1029
+ ...
1030
+
1031
+ @typing.overload
1032
+ def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1033
+ ...
1034
+
1035
+ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1036
+ """
1037
+ Specifies the Conda environment for the step.
1038
+
1039
+ Information in this decorator will augment any
1040
+ attributes set in the `@conda_base` flow-level decorator. Hence,
1041
+ you can use `@conda_base` to set packages required by all
1042
+ steps and use `@conda` to specify step-specific overrides.
1043
+
1044
+ Parameters
1045
+ ----------
1046
+ packages : Dict[str, str], default {}
1047
+ Packages to use for this step. The key is the name of the package
1048
+ and the value is the version to use.
1049
+ libraries : Dict[str, str], default {}
1050
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1051
+ python : str, optional, default None
1052
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1053
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1054
+ disabled : bool, default False
1055
+ If set to True, disables @conda.
1056
+ """
1057
+ ...
1058
+
836
1059
  @typing.overload
837
1060
  def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
838
1061
  """
@@ -910,26 +1133,62 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
910
1133
  """
911
1134
  ...
912
1135
 
913
- def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1136
+ @typing.overload
1137
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
914
1138
  """
915
- Specifies that this step should execute on Kubernetes.
916
-
917
- Parameters
918
- ----------
919
- cpu : int, default 1
920
- Number of CPUs required for this step. If `@resources` is
921
- also present, the maximum value from all decorators is used.
922
- memory : int, default 4096
923
- Memory size (in MB) required for this step. If
924
- `@resources` is also present, the maximum value from all decorators is
925
- used.
926
- disk : int, default 10240
927
- Disk size (in MB) required for this step. If
928
- `@resources` is also present, the maximum value from all decorators is
929
- used.
930
- image : str, optional, default None
931
- Docker image to use when launching on Kubernetes. If not specified, and
932
- METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
1139
+ Internal decorator to support Fast bakery
1140
+ """
1141
+ ...
1142
+
1143
+ @typing.overload
1144
+ def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1145
+ ...
1146
+
1147
+ def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1148
+ """
1149
+ Internal decorator to support Fast bakery
1150
+ """
1151
+ ...
1152
+
1153
+ @typing.overload
1154
+ def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1155
+ """
1156
+ Decorator prototype for all step decorators. This function gets specialized
1157
+ and imported for all decorators types by _import_plugin_decorators().
1158
+ """
1159
+ ...
1160
+
1161
+ @typing.overload
1162
+ def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1163
+ ...
1164
+
1165
+ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1166
+ """
1167
+ Decorator prototype for all step decorators. This function gets specialized
1168
+ and imported for all decorators types by _import_plugin_decorators().
1169
+ """
1170
+ ...
1171
+
1172
+ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1173
+ """
1174
+ Specifies that this step should execute on Kubernetes.
1175
+
1176
+ Parameters
1177
+ ----------
1178
+ cpu : int, default 1
1179
+ Number of CPUs required for this step. If `@resources` is
1180
+ also present, the maximum value from all decorators is used.
1181
+ memory : int, default 4096
1182
+ Memory size (in MB) required for this step. If
1183
+ `@resources` is also present, the maximum value from all decorators is
1184
+ used.
1185
+ disk : int, default 10240
1186
+ Disk size (in MB) required for this step. If
1187
+ `@resources` is also present, the maximum value from all decorators is
1188
+ used.
1189
+ image : str, optional, default None
1190
+ Docker image to use when launching on Kubernetes. If not specified, and
1191
+ METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
933
1192
  not, a default Docker image mapping to the current version of Python is used.
934
1193
  image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
935
1194
  If given, the imagePullPolicy to be applied to the Docker image of the step.
@@ -972,265 +1231,6 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
972
1231
  """
973
1232
  ...
974
1233
 
975
- @typing.overload
976
- def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
977
- """
978
- Decorator prototype for all step decorators. This function gets specialized
979
- and imported for all decorators types by _import_plugin_decorators().
980
- """
981
- ...
982
-
983
- @typing.overload
984
- def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
985
- ...
986
-
987
- def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
988
- """
989
- Decorator prototype for all step decorators. This function gets specialized
990
- and imported for all decorators types by _import_plugin_decorators().
991
- """
992
- ...
993
-
994
- @typing.overload
995
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
996
- """
997
- Specifies secrets to be retrieved and injected as environment variables prior to
998
- the execution of a step.
999
-
1000
- Parameters
1001
- ----------
1002
- sources : List[Union[str, Dict[str, Any]]], default: []
1003
- List of secret specs, defining how the secrets are to be retrieved
1004
- """
1005
- ...
1006
-
1007
- @typing.overload
1008
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1009
- ...
1010
-
1011
- @typing.overload
1012
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1013
- ...
1014
-
1015
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1016
- """
1017
- Specifies secrets to be retrieved and injected as environment variables prior to
1018
- the execution of a step.
1019
-
1020
- Parameters
1021
- ----------
1022
- sources : List[Union[str, Dict[str, Any]]], default: []
1023
- List of secret specs, defining how the secrets are to be retrieved
1024
- """
1025
- ...
1026
-
1027
- @typing.overload
1028
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1029
- """
1030
- Specifies the Conda environment for the step.
1031
-
1032
- Information in this decorator will augment any
1033
- attributes set in the `@conda_base` flow-level decorator. Hence,
1034
- you can use `@conda_base` to set packages required by all
1035
- steps and use `@conda` to specify step-specific overrides.
1036
-
1037
- Parameters
1038
- ----------
1039
- packages : Dict[str, str], default {}
1040
- Packages to use for this step. The key is the name of the package
1041
- and the value is the version to use.
1042
- libraries : Dict[str, str], default {}
1043
- Supported for backward compatibility. When used with packages, packages will take precedence.
1044
- python : str, optional, default None
1045
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1046
- that the version used will correspond to the version of the Python interpreter used to start the run.
1047
- disabled : bool, default False
1048
- If set to True, disables @conda.
1049
- """
1050
- ...
1051
-
1052
- @typing.overload
1053
- def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1054
- ...
1055
-
1056
- @typing.overload
1057
- def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1058
- ...
1059
-
1060
- def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1061
- """
1062
- Specifies the Conda environment for the step.
1063
-
1064
- Information in this decorator will augment any
1065
- attributes set in the `@conda_base` flow-level decorator. Hence,
1066
- you can use `@conda_base` to set packages required by all
1067
- steps and use `@conda` to specify step-specific overrides.
1068
-
1069
- Parameters
1070
- ----------
1071
- packages : Dict[str, str], default {}
1072
- Packages to use for this step. The key is the name of the package
1073
- and the value is the version to use.
1074
- libraries : Dict[str, str], default {}
1075
- Supported for backward compatibility. When used with packages, packages will take precedence.
1076
- python : str, optional, default None
1077
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1078
- that the version used will correspond to the version of the Python interpreter used to start the run.
1079
- disabled : bool, default False
1080
- If set to True, disables @conda.
1081
- """
1082
- ...
1083
-
1084
- @typing.overload
1085
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1086
- """
1087
- Specifies environment variables to be set prior to the execution of a step.
1088
-
1089
- Parameters
1090
- ----------
1091
- vars : Dict[str, str], default {}
1092
- Dictionary of environment variables to set.
1093
- """
1094
- ...
1095
-
1096
- @typing.overload
1097
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1098
- ...
1099
-
1100
- @typing.overload
1101
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1102
- ...
1103
-
1104
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1105
- """
1106
- Specifies environment variables to be set prior to the execution of a step.
1107
-
1108
- Parameters
1109
- ----------
1110
- vars : Dict[str, str], default {}
1111
- Dictionary of environment variables to set.
1112
- """
1113
- ...
1114
-
1115
- @typing.overload
1116
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1117
- """
1118
- Specifies the PyPI packages for the step.
1119
-
1120
- Information in this decorator will augment any
1121
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1122
- you can use `@pypi_base` to set packages required by all
1123
- steps and use `@pypi` to specify step-specific overrides.
1124
-
1125
- Parameters
1126
- ----------
1127
- packages : Dict[str, str], default: {}
1128
- Packages to use for this step. The key is the name of the package
1129
- and the value is the version to use.
1130
- python : str, optional, default: None
1131
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1132
- that the version used will correspond to the version of the Python interpreter used to start the run.
1133
- """
1134
- ...
1135
-
1136
- @typing.overload
1137
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1138
- ...
1139
-
1140
- @typing.overload
1141
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1142
- ...
1143
-
1144
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1145
- """
1146
- Specifies the PyPI packages for the step.
1147
-
1148
- Information in this decorator will augment any
1149
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1150
- you can use `@pypi_base` to set packages required by all
1151
- steps and use `@pypi` to specify step-specific overrides.
1152
-
1153
- Parameters
1154
- ----------
1155
- packages : Dict[str, str], default: {}
1156
- Packages to use for this step. The key is the name of the package
1157
- and the value is the version to use.
1158
- python : str, optional, default: None
1159
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1160
- that the version used will correspond to the version of the Python interpreter used to start the run.
1161
- """
1162
- ...
1163
-
1164
- @typing.overload
1165
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1166
- """
1167
- Internal decorator to support Fast bakery
1168
- """
1169
- ...
1170
-
1171
- @typing.overload
1172
- def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1173
- ...
1174
-
1175
- def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
1176
- """
1177
- Internal decorator to support Fast bakery
1178
- """
1179
- ...
1180
-
1181
- @typing.overload
1182
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1183
- """
1184
- Specifies the number of times the task corresponding
1185
- to a step needs to be retried.
1186
-
1187
- This decorator is useful for handling transient errors, such as networking issues.
1188
- If your task contains operations that can't be retried safely, e.g. database updates,
1189
- it is advisable to annotate it with `@retry(times=0)`.
1190
-
1191
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1192
- decorator will execute a no-op task after all retries have been exhausted,
1193
- ensuring that the flow execution can continue.
1194
-
1195
- Parameters
1196
- ----------
1197
- times : int, default 3
1198
- Number of times to retry this task.
1199
- minutes_between_retries : int, default 2
1200
- Number of minutes between retries.
1201
- """
1202
- ...
1203
-
1204
- @typing.overload
1205
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1206
- ...
1207
-
1208
- @typing.overload
1209
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1210
- ...
1211
-
1212
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1213
- """
1214
- Specifies the number of times the task corresponding
1215
- to a step needs to be retried.
1216
-
1217
- This decorator is useful for handling transient errors, such as networking issues.
1218
- If your task contains operations that can't be retried safely, e.g. database updates,
1219
- it is advisable to annotate it with `@retry(times=0)`.
1220
-
1221
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1222
- decorator will execute a no-op task after all retries have been exhausted,
1223
- ensuring that the flow execution can continue.
1224
-
1225
- Parameters
1226
- ----------
1227
- times : int, default 3
1228
- Number of times to retry this task.
1229
- minutes_between_retries : int, default 2
1230
- Number of minutes between retries.
1231
- """
1232
- ...
1233
-
1234
1234
  @typing.overload
1235
1235
  def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1236
1236
  """
@@ -1282,6 +1282,45 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
1282
1282
  """
1283
1283
  ...
1284
1284
 
1285
+ @typing.overload
1286
+ def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1287
+ """
1288
+ Specifies the PyPI packages for all steps of the flow.
1289
+
1290
+ Use `@pypi_base` to set common packages required by all
1291
+ steps and use `@pypi` to specify step-specific overrides.
1292
+ Parameters
1293
+ ----------
1294
+ packages : Dict[str, str], default: {}
1295
+ Packages to use for this flow. The key is the name of the package
1296
+ and the value is the version to use.
1297
+ python : str, optional, default: None
1298
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1299
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1300
+ """
1301
+ ...
1302
+
1303
+ @typing.overload
1304
+ def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1305
+ ...
1306
+
1307
+ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1308
+ """
1309
+ Specifies the PyPI packages for all steps of the flow.
1310
+
1311
+ Use `@pypi_base` to set common packages required by all
1312
+ steps and use `@pypi` to specify step-specific overrides.
1313
+ Parameters
1314
+ ----------
1315
+ packages : Dict[str, str], default: {}
1316
+ Packages to use for this flow. The key is the name of the package
1317
+ and the value is the version to use.
1318
+ python : str, optional, default: None
1319
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1320
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1321
+ """
1322
+ ...
1323
+
1285
1324
  @typing.overload
1286
1325
  def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1287
1326
  """
@@ -1385,37 +1424,6 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1385
1424
  """
1386
1425
  ...
1387
1426
 
1388
- def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1389
- """
1390
- This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1391
-
1392
- User code call
1393
- -----------
1394
- @nim(
1395
- models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1396
- backend='managed'
1397
- )
1398
-
1399
- Valid backend options
1400
- ---------------------
1401
- - 'managed': Outerbounds selects a compute provider based on the model.
1402
- - 🚧 'dataplane': Run in your account.
1403
-
1404
- Valid model options
1405
- ----------------
1406
- - 'meta/llama3-8b-instruct': 8B parameter model
1407
- - 'meta/llama3-70b-instruct': 70B parameter model
1408
- - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1409
-
1410
- Parameters
1411
- ----------
1412
- models: list[NIM]
1413
- List of NIM containers running models in sidecars.
1414
- backend: str
1415
- Compute provider to run the NIM container.
1416
- """
1417
- ...
1418
-
1419
1427
  def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1420
1428
  """
1421
1429
  The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
@@ -1458,55 +1466,6 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1458
1466
  """
1459
1467
  ...
1460
1468
 
1461
- @typing.overload
1462
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1463
- """
1464
- Specifies the Conda environment for all steps of the flow.
1465
-
1466
- Use `@conda_base` to set common libraries required by all
1467
- steps and use `@conda` to specify step-specific additions.
1468
-
1469
- Parameters
1470
- ----------
1471
- packages : Dict[str, str], default {}
1472
- Packages to use for this flow. The key is the name of the package
1473
- and the value is the version to use.
1474
- libraries : Dict[str, str], default {}
1475
- Supported for backward compatibility. When used with packages, packages will take precedence.
1476
- python : str, optional, default None
1477
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1478
- that the version used will correspond to the version of the Python interpreter used to start the run.
1479
- disabled : bool, default False
1480
- If set to True, disables Conda.
1481
- """
1482
- ...
1483
-
1484
- @typing.overload
1485
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1486
- ...
1487
-
1488
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1489
- """
1490
- Specifies the Conda environment for all steps of the flow.
1491
-
1492
- Use `@conda_base` to set common libraries required by all
1493
- steps and use `@conda` to specify step-specific additions.
1494
-
1495
- Parameters
1496
- ----------
1497
- packages : Dict[str, str], default {}
1498
- Packages to use for this flow. The key is the name of the package
1499
- and the value is the version to use.
1500
- libraries : Dict[str, str], default {}
1501
- Supported for backward compatibility. When used with packages, packages will take precedence.
1502
- python : str, optional, default None
1503
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1504
- that the version used will correspond to the version of the Python interpreter used to start the run.
1505
- disabled : bool, default False
1506
- If set to True, disables Conda.
1507
- """
1508
- ...
1509
-
1510
1469
  @typing.overload
1511
1470
  def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1512
1471
  """
@@ -1602,55 +1561,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
1602
1561
  """
1603
1562
  ...
1604
1563
 
1605
- @typing.overload
1606
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1607
- """
1608
- Specifies the times when the flow should be run when running on a
1609
- production scheduler.
1610
-
1611
- Parameters
1612
- ----------
1613
- hourly : bool, default False
1614
- Run the workflow hourly.
1615
- daily : bool, default True
1616
- Run the workflow daily.
1617
- weekly : bool, default False
1618
- Run the workflow weekly.
1619
- cron : str, optional, default None
1620
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1621
- specified by this expression.
1622
- timezone : str, optional, default None
1623
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1624
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1625
- """
1626
- ...
1627
-
1628
- @typing.overload
1629
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1630
- ...
1631
-
1632
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1633
- """
1634
- Specifies the times when the flow should be run when running on a
1635
- production scheduler.
1636
-
1637
- Parameters
1638
- ----------
1639
- hourly : bool, default False
1640
- Run the workflow hourly.
1641
- daily : bool, default True
1642
- Run the workflow daily.
1643
- weekly : bool, default False
1644
- Run the workflow weekly.
1645
- cron : str, optional, default None
1646
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1647
- specified by this expression.
1648
- timezone : str, optional, default None
1649
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1650
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1651
- """
1652
- ...
1653
-
1654
1564
  def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1655
1565
  """
1656
1566
  Specifies what flows belong to the same project.
@@ -1711,42 +1621,132 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1711
1621
  """
1712
1622
  ...
1713
1623
 
1624
+ def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1625
+ """
1626
+ This decorator is used to run NIM containers in Metaflow tasks as sidecars.
1627
+
1628
+ User code call
1629
+ -----------
1630
+ @nim(
1631
+ models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
1632
+ backend='managed'
1633
+ )
1634
+
1635
+ Valid backend options
1636
+ ---------------------
1637
+ - 'managed': Outerbounds selects a compute provider based on the model.
1638
+ - 🚧 'dataplane': Run in your account.
1639
+
1640
+ Valid model options
1641
+ ----------------
1642
+ - 'meta/llama3-8b-instruct': 8B parameter model
1643
+ - 'meta/llama3-70b-instruct': 70B parameter model
1644
+ - Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
1645
+
1646
+ Parameters
1647
+ ----------
1648
+ models: list[NIM]
1649
+ List of NIM containers running models in sidecars.
1650
+ backend: str
1651
+ Compute provider to run the NIM container.
1652
+ """
1653
+ ...
1654
+
1714
1655
  @typing.overload
1715
- def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1656
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1716
1657
  """
1717
- Specifies the PyPI packages for all steps of the flow.
1658
+ Specifies the times when the flow should be run when running on a
1659
+ production scheduler.
1718
1660
 
1719
- Use `@pypi_base` to set common packages required by all
1720
- steps and use `@pypi` to specify step-specific overrides.
1721
1661
  Parameters
1722
1662
  ----------
1723
- packages : Dict[str, str], default: {}
1663
+ hourly : bool, default False
1664
+ Run the workflow hourly.
1665
+ daily : bool, default True
1666
+ Run the workflow daily.
1667
+ weekly : bool, default False
1668
+ Run the workflow weekly.
1669
+ cron : str, optional, default None
1670
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1671
+ specified by this expression.
1672
+ timezone : str, optional, default None
1673
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1674
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1675
+ """
1676
+ ...
1677
+
1678
+ @typing.overload
1679
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1680
+ ...
1681
+
1682
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1683
+ """
1684
+ Specifies the times when the flow should be run when running on a
1685
+ production scheduler.
1686
+
1687
+ Parameters
1688
+ ----------
1689
+ hourly : bool, default False
1690
+ Run the workflow hourly.
1691
+ daily : bool, default True
1692
+ Run the workflow daily.
1693
+ weekly : bool, default False
1694
+ Run the workflow weekly.
1695
+ cron : str, optional, default None
1696
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1697
+ specified by this expression.
1698
+ timezone : str, optional, default None
1699
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1700
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1701
+ """
1702
+ ...
1703
+
1704
+ @typing.overload
1705
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1706
+ """
1707
+ Specifies the Conda environment for all steps of the flow.
1708
+
1709
+ Use `@conda_base` to set common libraries required by all
1710
+ steps and use `@conda` to specify step-specific additions.
1711
+
1712
+ Parameters
1713
+ ----------
1714
+ packages : Dict[str, str], default {}
1724
1715
  Packages to use for this flow. The key is the name of the package
1725
1716
  and the value is the version to use.
1726
- python : str, optional, default: None
1717
+ libraries : Dict[str, str], default {}
1718
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1719
+ python : str, optional, default None
1727
1720
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1728
1721
  that the version used will correspond to the version of the Python interpreter used to start the run.
1722
+ disabled : bool, default False
1723
+ If set to True, disables Conda.
1729
1724
  """
1730
1725
  ...
1731
1726
 
1732
1727
  @typing.overload
1733
- def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1728
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1734
1729
  ...
1735
1730
 
1736
- def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1731
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1737
1732
  """
1738
- Specifies the PyPI packages for all steps of the flow.
1733
+ Specifies the Conda environment for all steps of the flow.
1734
+
1735
+ Use `@conda_base` to set common libraries required by all
1736
+ steps and use `@conda` to specify step-specific additions.
1739
1737
 
1740
- Use `@pypi_base` to set common packages required by all
1741
- steps and use `@pypi` to specify step-specific overrides.
1742
1738
  Parameters
1743
1739
  ----------
1744
- packages : Dict[str, str], default: {}
1740
+ packages : Dict[str, str], default {}
1745
1741
  Packages to use for this flow. The key is the name of the package
1746
1742
  and the value is the version to use.
1747
- python : str, optional, default: None
1743
+ libraries : Dict[str, str], default {}
1744
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1745
+ python : str, optional, default None
1748
1746
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1749
1747
  that the version used will correspond to the version of the Python interpreter used to start the run.
1748
+ disabled : bool, default False
1749
+ If set to True, disables Conda.
1750
1750
  """
1751
1751
  ...
1752
1752