metaflow-stubs 2.11.15__py2.py3-none-any.whl → 2.11.16__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. metaflow-stubs/__init__.pyi +378 -378
  2. metaflow-stubs/cards.pyi +5 -5
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/client/__init__.pyi +3 -3
  5. metaflow-stubs/client/core.pyi +5 -5
  6. metaflow-stubs/client/filecache.pyi +3 -3
  7. metaflow-stubs/clone_util.pyi +2 -2
  8. metaflow-stubs/events.pyi +3 -3
  9. metaflow-stubs/exception.pyi +2 -2
  10. metaflow-stubs/flowspec.pyi +4 -4
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +5 -5
  13. metaflow-stubs/metadata/metadata.pyi +3 -3
  14. metaflow-stubs/metadata/util.pyi +2 -2
  15. metaflow-stubs/metaflow_config.pyi +14 -2
  16. metaflow-stubs/metaflow_current.pyi +4 -4
  17. metaflow-stubs/mflog/mflog.pyi +2 -2
  18. metaflow-stubs/multicore_utils.pyi +2 -2
  19. metaflow-stubs/parameters.pyi +4 -4
  20. metaflow-stubs/plugins/__init__.pyi +10 -2
  21. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  22. metaflow-stubs/plugins/airflow/airflow.pyi +12 -4
  23. metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
  24. metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
  25. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  26. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  27. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  28. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  29. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
  30. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
  31. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  32. metaflow-stubs/plugins/argo/argo_client.pyi +4 -2
  33. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  34. metaflow-stubs/plugins/argo/argo_workflows.pyi +8 -4
  35. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +3 -3
  36. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
  37. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  38. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  39. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  40. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  41. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  42. metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
  43. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  44. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
  45. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  46. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  47. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  48. metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
  49. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  50. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
  51. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  52. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  53. metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
  54. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  55. metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
  56. metaflow-stubs/plugins/azure/__init__.pyi +5 -2
  57. metaflow-stubs/plugins/azure/azure_credential.pyi +28 -0
  58. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  59. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +58 -0
  60. metaflow-stubs/plugins/azure/azure_utils.pyi +5 -5
  61. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +5 -5
  62. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  63. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  64. metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
  65. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  66. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  67. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  68. metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
  69. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  70. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  71. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  72. metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
  73. metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
  74. metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
  75. metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
  76. metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
  77. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  78. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  79. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  80. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
  81. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  82. metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
  83. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  84. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  85. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  86. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  87. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  88. metaflow-stubs/plugins/datatools/s3/s3.pyi +8 -5
  89. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  90. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  91. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  92. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  93. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  94. metaflow-stubs/plugins/events_decorator.pyi +3 -3
  95. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  96. metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
  97. metaflow-stubs/plugins/gcp/__init__.pyi +5 -2
  98. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +73 -0
  99. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  100. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +16 -2
  101. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  102. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  103. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  104. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +15 -3
  105. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +15 -4
  106. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +13 -2
  107. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +22 -3
  108. metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +15 -2
  109. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +127 -0
  110. metaflow-stubs/plugins/logs_cli.pyi +4 -4
  111. metaflow-stubs/plugins/package_cli.pyi +2 -2
  112. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  113. metaflow-stubs/plugins/project_decorator.pyi +3 -3
  114. metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
  115. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  116. metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
  117. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  118. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  119. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  120. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  121. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  122. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  123. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  124. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  125. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  126. metaflow-stubs/plugins/tag_cli.pyi +3 -3
  127. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  128. metaflow-stubs/plugins/timeout_decorator.pyi +4 -2
  129. metaflow-stubs/procpoll.pyi +2 -2
  130. metaflow-stubs/pylint_wrapper.pyi +2 -2
  131. metaflow-stubs/tagging_util.pyi +2 -2
  132. metaflow-stubs/tuple_util.pyi +14 -0
  133. metaflow-stubs/version.pyi +2 -2
  134. {metaflow_stubs-2.11.15.dist-info → metaflow_stubs-2.11.16.dist-info}/METADATA +2 -2
  135. metaflow_stubs-2.11.16.dist-info/RECORD +138 -0
  136. metaflow_stubs-2.11.15.dist-info/RECORD +0 -133
  137. {metaflow_stubs-2.11.15.dist-info → metaflow_stubs-2.11.16.dist-info}/WHEEL +0 -0
  138. {metaflow_stubs-2.11.15.dist-info → metaflow_stubs-2.11.16.dist-info}/top_level.txt +0 -0
@@ -1,23 +1,23 @@
1
1
  ##################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.11.15 #
4
- # Generated on 2024-05-08T22:46:02.315605 #
3
+ # MF version: 2.11.16 #
4
+ # Generated on 2024-05-22T00:31:19.988196 #
5
5
  ##################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
+ import metaflow._vendor.click.types
12
+ import metaflow.plugins.datatools.s3.s3
11
13
  import metaflow.events
14
+ import metaflow.datastore.inputs
15
+ import typing
12
16
  import datetime
13
- import metaflow.parameters
14
- import io
15
17
  import metaflow.client.core
16
- import metaflow.datastore.inputs
17
- import metaflow.plugins.datatools.s3.s3
18
+ import io
18
19
  import metaflow.metaflow_current
19
- import metaflow._vendor.click.types
20
- import typing
20
+ import metaflow.parameters
21
21
  FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
22
22
  StepFlag = typing.NewType("StepFlag", bool)
23
23
 
@@ -802,6 +802,57 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
802
802
  """
803
803
  ...
804
804
 
805
+ @typing.overload
806
+ def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
807
+ """
808
+ Creates a human-readable report, a Metaflow Card, after this step completes.
809
+
810
+ Note that you may add multiple `@card` decorators in a step with different parameters.
811
+
812
+ Parameters
813
+ ----------
814
+ type : str, default 'default'
815
+ Card type.
816
+ id : str, optional, default None
817
+ If multiple cards are present, use this id to identify this card.
818
+ options : Dict[str, Any], default {}
819
+ Options passed to the card. The contents depend on the card type.
820
+ timeout : int, default 45
821
+ Interrupt reporting if it takes more than this many seconds.
822
+
823
+
824
+ """
825
+ ...
826
+
827
+ @typing.overload
828
+ def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
829
+ ...
830
+
831
+ @typing.overload
832
+ def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
833
+ ...
834
+
835
+ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
836
+ """
837
+ Creates a human-readable report, a Metaflow Card, after this step completes.
838
+
839
+ Note that you may add multiple `@card` decorators in a step with different parameters.
840
+
841
+ Parameters
842
+ ----------
843
+ type : str, default 'default'
844
+ Card type.
845
+ id : str, optional, default None
846
+ If multiple cards are present, use this id to identify this card.
847
+ options : Dict[str, Any], default {}
848
+ Options passed to the card. The contents depend on the card type.
849
+ timeout : int, default 45
850
+ Interrupt reporting if it takes more than this many seconds.
851
+
852
+
853
+ """
854
+ ...
855
+
805
856
  @typing.overload
806
857
  def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
807
858
  """
@@ -1009,159 +1060,33 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
1009
1060
  ...
1010
1061
 
1011
1062
  @typing.overload
1012
- def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1013
- """
1014
- Creates a human-readable report, a Metaflow Card, after this step completes.
1015
-
1016
- Note that you may add multiple `@card` decorators in a step with different parameters.
1017
-
1018
- Parameters
1019
- ----------
1020
- type : str, default 'default'
1021
- Card type.
1022
- id : str, optional, default None
1023
- If multiple cards are present, use this id to identify this card.
1024
- options : Dict[str, Any], default {}
1025
- Options passed to the card. The contents depend on the card type.
1026
- timeout : int, default 45
1027
- Interrupt reporting if it takes more than this many seconds.
1028
-
1029
-
1030
- """
1031
- ...
1032
-
1033
- @typing.overload
1034
- def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1035
- ...
1036
-
1037
- @typing.overload
1038
- def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1039
- ...
1040
-
1041
- def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
1042
- """
1043
- Creates a human-readable report, a Metaflow Card, after this step completes.
1044
-
1045
- Note that you may add multiple `@card` decorators in a step with different parameters.
1046
-
1047
- Parameters
1048
- ----------
1049
- type : str, default 'default'
1050
- Card type.
1051
- id : str, optional, default None
1052
- If multiple cards are present, use this id to identify this card.
1053
- options : Dict[str, Any], default {}
1054
- Options passed to the card. The contents depend on the card type.
1055
- timeout : int, default 45
1056
- Interrupt reporting if it takes more than this many seconds.
1057
-
1058
-
1059
- """
1060
- ...
1061
-
1062
- @typing.overload
1063
- def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1064
- """
1065
- Specifies the PyPI packages for the step.
1066
-
1067
- Information in this decorator will augment any
1068
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1069
- you can use `@pypi_base` to set packages required by all
1070
- steps and use `@pypi` to specify step-specific overrides.
1071
-
1072
- Parameters
1073
- ----------
1074
- packages : Dict[str, str], default: {}
1075
- Packages to use for this step. The key is the name of the package
1076
- and the value is the version to use.
1077
- python : str, optional, default: None
1078
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1079
- that the version used will correspond to the version of the Python interpreter used to start the run.
1080
- """
1081
- ...
1082
-
1083
- @typing.overload
1084
- def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1085
- ...
1086
-
1087
- @typing.overload
1088
- def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1089
- ...
1090
-
1091
- def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1092
- """
1093
- Specifies the PyPI packages for the step.
1094
-
1095
- Information in this decorator will augment any
1096
- attributes set in the `@pyi_base` flow-level decorator. Hence,
1097
- you can use `@pypi_base` to set packages required by all
1098
- steps and use `@pypi` to specify step-specific overrides.
1099
-
1100
- Parameters
1101
- ----------
1102
- packages : Dict[str, str], default: {}
1103
- Packages to use for this step. The key is the name of the package
1104
- and the value is the version to use.
1105
- python : str, optional, default: None
1106
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1107
- that the version used will correspond to the version of the Python interpreter used to start the run.
1108
- """
1109
- ...
1110
-
1111
- @typing.overload
1112
- def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1063
+ def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1113
1064
  """
1114
- Specifies a timeout for your step.
1115
-
1116
- This decorator is useful if this step may hang indefinitely.
1117
-
1118
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1119
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1120
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1121
-
1122
- Note that all the values specified in parameters are added together so if you specify
1123
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1065
+ Specifies environment variables to be set prior to the execution of a step.
1124
1066
 
1125
1067
  Parameters
1126
1068
  ----------
1127
- seconds : int, default 0
1128
- Number of seconds to wait prior to timing out.
1129
- minutes : int, default 0
1130
- Number of minutes to wait prior to timing out.
1131
- hours : int, default 0
1132
- Number of hours to wait prior to timing out.
1069
+ vars : Dict[str, str], default {}
1070
+ Dictionary of environment variables to set.
1133
1071
  """
1134
1072
  ...
1135
1073
 
1136
1074
  @typing.overload
1137
- def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1075
+ def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1138
1076
  ...
1139
1077
 
1140
1078
  @typing.overload
1141
- def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1079
+ def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1142
1080
  ...
1143
1081
 
1144
- def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1082
+ def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1145
1083
  """
1146
- Specifies a timeout for your step.
1147
-
1148
- This decorator is useful if this step may hang indefinitely.
1149
-
1150
- This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1151
- A timeout is considered to be an exception thrown by the step. It will cause the step to be
1152
- retried if needed and the exception will be caught by the `@catch` decorator, if present.
1153
-
1154
- Note that all the values specified in parameters are added together so if you specify
1155
- 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1084
+ Specifies environment variables to be set prior to the execution of a step.
1156
1085
 
1157
1086
  Parameters
1158
1087
  ----------
1159
- seconds : int, default 0
1160
- Number of seconds to wait prior to timing out.
1161
- minutes : int, default 0
1162
- Number of minutes to wait prior to timing out.
1163
- hours : int, default 0
1164
- Number of hours to wait prior to timing out.
1088
+ vars : Dict[str, str], default {}
1089
+ Dictionary of environment variables to set.
1165
1090
  """
1166
1091
  ...
1167
1092
 
@@ -1215,42 +1140,95 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1215
1140
  ...
1216
1141
 
1217
1142
  @typing.overload
1218
- def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1143
+ def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1219
1144
  """
1220
- Specifies secrets to be retrieved and injected as environment variables prior to
1221
- the execution of a step.
1145
+ Specifies the number of times the task corresponding
1146
+ to a step needs to be retried.
1147
+
1148
+ This decorator is useful for handling transient errors, such as networking issues.
1149
+ If your task contains operations that can't be retried safely, e.g. database updates,
1150
+ it is advisable to annotate it with `@retry(times=0)`.
1151
+
1152
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1153
+ decorator will execute a no-op task after all retries have been exhausted,
1154
+ ensuring that the flow execution can continue.
1222
1155
 
1223
1156
  Parameters
1224
1157
  ----------
1225
- sources : List[Union[str, Dict[str, Any]]], default: []
1226
- List of secret specs, defining how the secrets are to be retrieved
1158
+ times : int, default 3
1159
+ Number of times to retry this task.
1160
+ minutes_between_retries : int, default 2
1161
+ Number of minutes between retries.
1227
1162
  """
1228
1163
  ...
1229
1164
 
1230
1165
  @typing.overload
1231
- def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1166
+ def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1232
1167
  ...
1233
1168
 
1234
1169
  @typing.overload
1235
- def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1170
+ def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1236
1171
  ...
1237
1172
 
1238
- def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1173
+ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1239
1174
  """
1240
- Specifies secrets to be retrieved and injected as environment variables prior to
1241
- the execution of a step.
1175
+ Specifies the number of times the task corresponding
1176
+ to a step needs to be retried.
1177
+
1178
+ This decorator is useful for handling transient errors, such as networking issues.
1179
+ If your task contains operations that can't be retried safely, e.g. database updates,
1180
+ it is advisable to annotate it with `@retry(times=0)`.
1181
+
1182
+ This can be used in conjunction with the `@catch` decorator. The `@catch`
1183
+ decorator will execute a no-op task after all retries have been exhausted,
1184
+ ensuring that the flow execution can continue.
1242
1185
 
1243
1186
  Parameters
1244
1187
  ----------
1245
- sources : List[Union[str, Dict[str, Any]]], default: []
1246
- List of secret specs, defining how the secrets are to be retrieved
1188
+ times : int, default 3
1189
+ Number of times to retry this task.
1190
+ minutes_between_retries : int, default 2
1191
+ Number of minutes between retries.
1247
1192
  """
1248
1193
  ...
1249
1194
 
1250
1195
  @typing.overload
1251
- def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1196
+ def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1252
1197
  """
1253
- Specifies the Conda environment for the step.
1198
+ Specifies secrets to be retrieved and injected as environment variables prior to
1199
+ the execution of a step.
1200
+
1201
+ Parameters
1202
+ ----------
1203
+ sources : List[Union[str, Dict[str, Any]]], default: []
1204
+ List of secret specs, defining how the secrets are to be retrieved
1205
+ """
1206
+ ...
1207
+
1208
+ @typing.overload
1209
+ def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1210
+ ...
1211
+
1212
+ @typing.overload
1213
+ def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1214
+ ...
1215
+
1216
+ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
1217
+ """
1218
+ Specifies secrets to be retrieved and injected as environment variables prior to
1219
+ the execution of a step.
1220
+
1221
+ Parameters
1222
+ ----------
1223
+ sources : List[Union[str, Dict[str, Any]]], default: []
1224
+ List of secret specs, defining how the secrets are to be retrieved
1225
+ """
1226
+ ...
1227
+
1228
+ @typing.overload
1229
+ def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1230
+ """
1231
+ Specifies the Conda environment for the step.
1254
1232
 
1255
1233
  Information in this decorator will augment any
1256
1234
  attributes set in the `@conda_base` flow-level decorator. Hence,
@@ -1305,230 +1283,108 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
1305
1283
  ...
1306
1284
 
1307
1285
  @typing.overload
1308
- def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1309
- """
1310
- Specifies environment variables to be set prior to the execution of a step.
1311
-
1312
- Parameters
1313
- ----------
1314
- vars : Dict[str, str], default {}
1315
- Dictionary of environment variables to set.
1316
- """
1317
- ...
1318
-
1319
- @typing.overload
1320
- def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1321
- ...
1322
-
1323
- @typing.overload
1324
- def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1325
- ...
1326
-
1327
- def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
1286
+ def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1328
1287
  """
1329
- Specifies environment variables to be set prior to the execution of a step.
1288
+ Specifies a timeout for your step.
1330
1289
 
1331
- Parameters
1332
- ----------
1333
- vars : Dict[str, str], default {}
1334
- Dictionary of environment variables to set.
1335
- """
1336
- ...
1337
-
1338
- @typing.overload
1339
- def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1340
- """
1341
- Specifies the number of times the task corresponding
1342
- to a step needs to be retried.
1290
+ This decorator is useful if this step may hang indefinitely.
1343
1291
 
1344
- This decorator is useful for handling transient errors, such as networking issues.
1345
- If your task contains operations that can't be retried safely, e.g. database updates,
1346
- it is advisable to annotate it with `@retry(times=0)`.
1292
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1293
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1294
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1347
1295
 
1348
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1349
- decorator will execute a no-op task after all retries have been exhausted,
1350
- ensuring that the flow execution can continue.
1296
+ Note that all the values specified in parameters are added together so if you specify
1297
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1351
1298
 
1352
1299
  Parameters
1353
1300
  ----------
1354
- times : int, default 3
1355
- Number of times to retry this task.
1356
- minutes_between_retries : int, default 2
1357
- Number of minutes between retries.
1301
+ seconds : int, default 0
1302
+ Number of seconds to wait prior to timing out.
1303
+ minutes : int, default 0
1304
+ Number of minutes to wait prior to timing out.
1305
+ hours : int, default 0
1306
+ Number of hours to wait prior to timing out.
1358
1307
  """
1359
1308
  ...
1360
1309
 
1361
1310
  @typing.overload
1362
- def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1311
+ def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1363
1312
  ...
1364
1313
 
1365
1314
  @typing.overload
1366
- def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1315
+ def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1367
1316
  ...
1368
1317
 
1369
- def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
1318
+ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
1370
1319
  """
1371
- Specifies the number of times the task corresponding
1372
- to a step needs to be retried.
1373
-
1374
- This decorator is useful for handling transient errors, such as networking issues.
1375
- If your task contains operations that can't be retried safely, e.g. database updates,
1376
- it is advisable to annotate it with `@retry(times=0)`.
1320
+ Specifies a timeout for your step.
1377
1321
 
1378
- This can be used in conjunction with the `@catch` decorator. The `@catch`
1379
- decorator will execute a no-op task after all retries have been exhausted,
1380
- ensuring that the flow execution can continue.
1322
+ This decorator is useful if this step may hang indefinitely.
1381
1323
 
1382
- Parameters
1383
- ----------
1384
- times : int, default 3
1385
- Number of times to retry this task.
1386
- minutes_between_retries : int, default 2
1387
- Number of minutes between retries.
1388
- """
1389
- ...
1390
-
1391
- @typing.overload
1392
- def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1393
- """
1394
- Specifies the Conda environment for all steps of the flow.
1324
+ This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
1325
+ A timeout is considered to be an exception thrown by the step. It will cause the step to be
1326
+ retried if needed and the exception will be caught by the `@catch` decorator, if present.
1395
1327
 
1396
- Use `@conda_base` to set common libraries required by all
1397
- steps and use `@conda` to specify step-specific additions.
1328
+ Note that all the values specified in parameters are added together so if you specify
1329
+ 60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
1398
1330
 
1399
1331
  Parameters
1400
1332
  ----------
1401
- packages : Dict[str, str], default {}
1402
- Packages to use for this flow. The key is the name of the package
1403
- and the value is the version to use.
1404
- libraries : Dict[str, str], default {}
1405
- Supported for backward compatibility. When used with packages, packages will take precedence.
1406
- python : str, optional, default None
1407
- Version of Python to use, e.g. '3.7.4'. A default value of None implies
1408
- that the version used will correspond to the version of the Python interpreter used to start the run.
1409
- disabled : bool, default False
1410
- If set to True, disables Conda.
1333
+ seconds : int, default 0
1334
+ Number of seconds to wait prior to timing out.
1335
+ minutes : int, default 0
1336
+ Number of minutes to wait prior to timing out.
1337
+ hours : int, default 0
1338
+ Number of hours to wait prior to timing out.
1411
1339
  """
1412
1340
  ...
1413
1341
 
1414
1342
  @typing.overload
1415
- def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1416
- ...
1417
-
1418
- def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1343
+ def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
1419
1344
  """
1420
- Specifies the Conda environment for all steps of the flow.
1345
+ Specifies the PyPI packages for the step.
1421
1346
 
1422
- Use `@conda_base` to set common libraries required by all
1423
- steps and use `@conda` to specify step-specific additions.
1347
+ Information in this decorator will augment any
1348
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1349
+ you can use `@pypi_base` to set packages required by all
1350
+ steps and use `@pypi` to specify step-specific overrides.
1424
1351
 
1425
1352
  Parameters
1426
1353
  ----------
1427
- packages : Dict[str, str], default {}
1428
- Packages to use for this flow. The key is the name of the package
1354
+ packages : Dict[str, str], default: {}
1355
+ Packages to use for this step. The key is the name of the package
1429
1356
  and the value is the version to use.
1430
- libraries : Dict[str, str], default {}
1431
- Supported for backward compatibility. When used with packages, packages will take precedence.
1432
- python : str, optional, default None
1357
+ python : str, optional, default: None
1433
1358
  Version of Python to use, e.g. '3.7.4'. A default value of None implies
1434
1359
  that the version used will correspond to the version of the Python interpreter used to start the run.
1435
- disabled : bool, default False
1436
- If set to True, disables Conda.
1437
1360
  """
1438
1361
  ...
1439
1362
 
1440
1363
  @typing.overload
1441
- def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1442
- """
1443
- Specifies the event(s) that this flow depends on.
1444
-
1445
- ```
1446
- @trigger(event='foo')
1447
- ```
1448
- or
1449
- ```
1450
- @trigger(events=['foo', 'bar'])
1451
- ```
1452
-
1453
- Additionally, you can specify the parameter mappings
1454
- to map event payload to Metaflow parameters for the flow.
1455
- ```
1456
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1457
- ```
1458
- or
1459
- ```
1460
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1461
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1462
- ```
1463
-
1464
- 'parameters' can also be a list of strings and tuples like so:
1465
- ```
1466
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1467
- ```
1468
- This is equivalent to:
1469
- ```
1470
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1471
- ```
1472
-
1473
- Parameters
1474
- ----------
1475
- event : Union[str, Dict[str, Any]], optional, default None
1476
- Event dependency for this flow.
1477
- events : List[Union[str, Dict[str, Any]]], default []
1478
- Events dependency for this flow.
1479
- options : Dict[str, Any], default {}
1480
- Backend-specific configuration for tuning eventing behavior.
1481
-
1482
-
1483
- """
1364
+ def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
1484
1365
  ...
1485
1366
 
1486
1367
  @typing.overload
1487
- def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1368
+ def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
1488
1369
  ...
1489
1370
 
1490
- def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1371
+ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
1491
1372
  """
1492
- Specifies the event(s) that this flow depends on.
1493
-
1494
- ```
1495
- @trigger(event='foo')
1496
- ```
1497
- or
1498
- ```
1499
- @trigger(events=['foo', 'bar'])
1500
- ```
1501
-
1502
- Additionally, you can specify the parameter mappings
1503
- to map event payload to Metaflow parameters for the flow.
1504
- ```
1505
- @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1506
- ```
1507
- or
1508
- ```
1509
- @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1510
- {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1511
- ```
1373
+ Specifies the PyPI packages for the step.
1512
1374
 
1513
- 'parameters' can also be a list of strings and tuples like so:
1514
- ```
1515
- @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1516
- ```
1517
- This is equivalent to:
1518
- ```
1519
- @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1520
- ```
1375
+ Information in this decorator will augment any
1376
+ attributes set in the `@pyi_base` flow-level decorator. Hence,
1377
+ you can use `@pypi_base` to set packages required by all
1378
+ steps and use `@pypi` to specify step-specific overrides.
1521
1379
 
1522
1380
  Parameters
1523
1381
  ----------
1524
- event : Union[str, Dict[str, Any]], optional, default None
1525
- Event dependency for this flow.
1526
- events : List[Union[str, Dict[str, Any]]], default []
1527
- Events dependency for this flow.
1528
- options : Dict[str, Any], default {}
1529
- Backend-specific configuration for tuning eventing behavior.
1530
-
1531
-
1382
+ packages : Dict[str, str], default: {}
1383
+ Packages to use for this step. The key is the name of the package
1384
+ and the value is the version to use.
1385
+ python : str, optional, default: None
1386
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1387
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1532
1388
  """
1533
1389
  ...
1534
1390
 
@@ -1613,6 +1469,55 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
1613
1469
  """
1614
1470
  ...
1615
1471
 
1472
+ @typing.overload
1473
+ def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1474
+ """
1475
+ Specifies the Conda environment for all steps of the flow.
1476
+
1477
+ Use `@conda_base` to set common libraries required by all
1478
+ steps and use `@conda` to specify step-specific additions.
1479
+
1480
+ Parameters
1481
+ ----------
1482
+ packages : Dict[str, str], default {}
1483
+ Packages to use for this flow. The key is the name of the package
1484
+ and the value is the version to use.
1485
+ libraries : Dict[str, str], default {}
1486
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1487
+ python : str, optional, default None
1488
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1489
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1490
+ disabled : bool, default False
1491
+ If set to True, disables Conda.
1492
+ """
1493
+ ...
1494
+
1495
+ @typing.overload
1496
+ def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1497
+ ...
1498
+
1499
+ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
1500
+ """
1501
+ Specifies the Conda environment for all steps of the flow.
1502
+
1503
+ Use `@conda_base` to set common libraries required by all
1504
+ steps and use `@conda` to specify step-specific additions.
1505
+
1506
+ Parameters
1507
+ ----------
1508
+ packages : Dict[str, str], default {}
1509
+ Packages to use for this flow. The key is the name of the package
1510
+ and the value is the version to use.
1511
+ libraries : Dict[str, str], default {}
1512
+ Supported for backward compatibility. When used with packages, packages will take precedence.
1513
+ python : str, optional, default None
1514
+ Version of Python to use, e.g. '3.7.4'. A default value of None implies
1515
+ that the version used will correspond to the version of the Python interpreter used to start the run.
1516
+ disabled : bool, default False
1517
+ If set to True, disables Conda.
1518
+ """
1519
+ ...
1520
+
1616
1521
  def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1617
1522
  """
1618
1523
  The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
@@ -1656,67 +1561,95 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
1656
1561
  ...
1657
1562
 
1658
1563
  @typing.overload
1659
- def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1564
+ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1660
1565
  """
1661
- Specifies the times when the flow should be run when running on a
1662
- production scheduler.
1566
+ Specifies the event(s) that this flow depends on.
1567
+
1568
+ ```
1569
+ @trigger(event='foo')
1570
+ ```
1571
+ or
1572
+ ```
1573
+ @trigger(events=['foo', 'bar'])
1574
+ ```
1575
+
1576
+ Additionally, you can specify the parameter mappings
1577
+ to map event payload to Metaflow parameters for the flow.
1578
+ ```
1579
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1580
+ ```
1581
+ or
1582
+ ```
1583
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1584
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1585
+ ```
1586
+
1587
+ 'parameters' can also be a list of strings and tuples like so:
1588
+ ```
1589
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1590
+ ```
1591
+ This is equivalent to:
1592
+ ```
1593
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1594
+ ```
1663
1595
 
1664
1596
  Parameters
1665
1597
  ----------
1666
- hourly : bool, default False
1667
- Run the workflow hourly.
1668
- daily : bool, default True
1669
- Run the workflow daily.
1670
- weekly : bool, default False
1671
- Run the workflow weekly.
1672
- cron : str, optional, default None
1673
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1674
- specified by this expression.
1675
- timezone : str, optional, default None
1676
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1677
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1598
+ event : Union[str, Dict[str, Any]], optional, default None
1599
+ Event dependency for this flow.
1600
+ events : List[Union[str, Dict[str, Any]]], default []
1601
+ Events dependency for this flow.
1602
+ options : Dict[str, Any], default {}
1603
+ Backend-specific configuration for tuning eventing behavior.
1604
+
1605
+
1678
1606
  """
1679
1607
  ...
1680
1608
 
1681
1609
  @typing.overload
1682
- def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1610
+ def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1683
1611
  ...
1684
1612
 
1685
- def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1613
+ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
1686
1614
  """
1687
- Specifies the times when the flow should be run when running on a
1688
- production scheduler.
1615
+ Specifies the event(s) that this flow depends on.
1689
1616
 
1690
- Parameters
1691
- ----------
1692
- hourly : bool, default False
1693
- Run the workflow hourly.
1694
- daily : bool, default True
1695
- Run the workflow daily.
1696
- weekly : bool, default False
1697
- Run the workflow weekly.
1698
- cron : str, optional, default None
1699
- Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1700
- specified by this expression.
1701
- timezone : str, optional, default None
1702
- Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1703
- which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1704
- """
1705
- ...
1706
-
1707
- def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1708
- """
1709
- Specifies what flows belong to the same project.
1617
+ ```
1618
+ @trigger(event='foo')
1619
+ ```
1620
+ or
1621
+ ```
1622
+ @trigger(events=['foo', 'bar'])
1623
+ ```
1710
1624
 
1711
- A project-specific namespace is created for all flows that
1712
- use the same `@project(name)`.
1625
+ Additionally, you can specify the parameter mappings
1626
+ to map event payload to Metaflow parameters for the flow.
1627
+ ```
1628
+ @trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
1629
+ ```
1630
+ or
1631
+ ```
1632
+ @trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
1633
+ {'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
1634
+ ```
1635
+
1636
+ 'parameters' can also be a list of strings and tuples like so:
1637
+ ```
1638
+ @trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
1639
+ ```
1640
+ This is equivalent to:
1641
+ ```
1642
+ @trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
1643
+ ```
1713
1644
 
1714
1645
  Parameters
1715
1646
  ----------
1716
- name : str
1717
- Project name. Make sure that the name is unique amongst all
1718
- projects that use the same production scheduler. The name may
1719
- contain only lowercase alphanumeric characters and underscores.
1647
+ event : Union[str, Dict[str, Any]], optional, default None
1648
+ Event dependency for this flow.
1649
+ events : List[Union[str, Dict[str, Any]]], default []
1650
+ Events dependency for this flow.
1651
+ options : Dict[str, Any], default {}
1652
+ Backend-specific configuration for tuning eventing behavior.
1720
1653
 
1721
1654
 
1722
1655
  """
@@ -1825,6 +1758,73 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
1825
1758
  """
1826
1759
  ...
1827
1760
 
1761
+ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1762
+ """
1763
+ Specifies what flows belong to the same project.
1764
+
1765
+ A project-specific namespace is created for all flows that
1766
+ use the same `@project(name)`.
1767
+
1768
+ Parameters
1769
+ ----------
1770
+ name : str
1771
+ Project name. Make sure that the name is unique amongst all
1772
+ projects that use the same production scheduler. The name may
1773
+ contain only lowercase alphanumeric characters and underscores.
1774
+
1775
+
1776
+ """
1777
+ ...
1778
+
1779
+ @typing.overload
1780
+ def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
1781
+ """
1782
+ Specifies the times when the flow should be run when running on a
1783
+ production scheduler.
1784
+
1785
+ Parameters
1786
+ ----------
1787
+ hourly : bool, default False
1788
+ Run the workflow hourly.
1789
+ daily : bool, default True
1790
+ Run the workflow daily.
1791
+ weekly : bool, default False
1792
+ Run the workflow weekly.
1793
+ cron : str, optional, default None
1794
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1795
+ specified by this expression.
1796
+ timezone : str, optional, default None
1797
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1798
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1799
+ """
1800
+ ...
1801
+
1802
+ @typing.overload
1803
+ def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
1804
+ ...
1805
+
1806
+ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
1807
+ """
1808
+ Specifies the times when the flow should be run when running on a
1809
+ production scheduler.
1810
+
1811
+ Parameters
1812
+ ----------
1813
+ hourly : bool, default False
1814
+ Run the workflow hourly.
1815
+ daily : bool, default True
1816
+ Run the workflow daily.
1817
+ weekly : bool, default False
1818
+ Run the workflow weekly.
1819
+ cron : str, optional, default None
1820
+ Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
1821
+ specified by this expression.
1822
+ timezone : str, optional, default None
1823
+ Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
1824
+ which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
1825
+ """
1826
+ ...
1827
+
1828
1828
  def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
1829
1829
  """
1830
1830
  Switch namespace to the one provided.