metaflow-stubs 2.11.15__py2.py3-none-any.whl → 2.11.16__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +378 -378
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +3 -3
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +5 -5
- metaflow-stubs/metadata/metadata.pyi +3 -3
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +14 -2
- metaflow-stubs/metaflow_current.pyi +4 -4
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +10 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +12 -4
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +4 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +8 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +5 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +28 -0
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +58 -0
- metaflow-stubs/plugins/azure/azure_utils.pyi +5 -5
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +5 -5
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +8 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +5 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +73 -0
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +16 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +15 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +15 -4
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +13 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +22 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +15 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +127 -0
- metaflow-stubs/plugins/logs_cli.pyi +4 -4
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +3 -3
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +4 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +14 -0
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.11.15.dist-info → metaflow_stubs-2.11.16.dist-info}/METADATA +2 -2
- metaflow_stubs-2.11.16.dist-info/RECORD +138 -0
- metaflow_stubs-2.11.15.dist-info/RECORD +0 -133
- {metaflow_stubs-2.11.15.dist-info → metaflow_stubs-2.11.16.dist-info}/WHEEL +0 -0
- {metaflow_stubs-2.11.15.dist-info → metaflow_stubs-2.11.16.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,23 +1,23 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.11.
|
4
|
-
# Generated on 2024-05-
|
3
|
+
# MF version: 2.11.16 #
|
4
|
+
# Generated on 2024-05-22T00:31:19.988196 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
+
import metaflow._vendor.click.types
|
12
|
+
import metaflow.plugins.datatools.s3.s3
|
11
13
|
import metaflow.events
|
14
|
+
import metaflow.datastore.inputs
|
15
|
+
import typing
|
12
16
|
import datetime
|
13
|
-
import metaflow.parameters
|
14
|
-
import io
|
15
17
|
import metaflow.client.core
|
16
|
-
import
|
17
|
-
import metaflow.plugins.datatools.s3.s3
|
18
|
+
import io
|
18
19
|
import metaflow.metaflow_current
|
19
|
-
import metaflow.
|
20
|
-
import typing
|
20
|
+
import metaflow.parameters
|
21
21
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
22
22
|
StepFlag = typing.NewType("StepFlag", bool)
|
23
23
|
|
@@ -802,6 +802,57 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
802
802
|
"""
|
803
803
|
...
|
804
804
|
|
805
|
+
@typing.overload
|
806
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
807
|
+
"""
|
808
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
809
|
+
|
810
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
811
|
+
|
812
|
+
Parameters
|
813
|
+
----------
|
814
|
+
type : str, default 'default'
|
815
|
+
Card type.
|
816
|
+
id : str, optional, default None
|
817
|
+
If multiple cards are present, use this id to identify this card.
|
818
|
+
options : Dict[str, Any], default {}
|
819
|
+
Options passed to the card. The contents depend on the card type.
|
820
|
+
timeout : int, default 45
|
821
|
+
Interrupt reporting if it takes more than this many seconds.
|
822
|
+
|
823
|
+
|
824
|
+
"""
|
825
|
+
...
|
826
|
+
|
827
|
+
@typing.overload
|
828
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
829
|
+
...
|
830
|
+
|
831
|
+
@typing.overload
|
832
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
833
|
+
...
|
834
|
+
|
835
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
836
|
+
"""
|
837
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
838
|
+
|
839
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
840
|
+
|
841
|
+
Parameters
|
842
|
+
----------
|
843
|
+
type : str, default 'default'
|
844
|
+
Card type.
|
845
|
+
id : str, optional, default None
|
846
|
+
If multiple cards are present, use this id to identify this card.
|
847
|
+
options : Dict[str, Any], default {}
|
848
|
+
Options passed to the card. The contents depend on the card type.
|
849
|
+
timeout : int, default 45
|
850
|
+
Interrupt reporting if it takes more than this many seconds.
|
851
|
+
|
852
|
+
|
853
|
+
"""
|
854
|
+
...
|
855
|
+
|
805
856
|
@typing.overload
|
806
857
|
def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
807
858
|
"""
|
@@ -1009,159 +1060,33 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
1009
1060
|
...
|
1010
1061
|
|
1011
1062
|
@typing.overload
|
1012
|
-
def
|
1013
|
-
"""
|
1014
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1015
|
-
|
1016
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1017
|
-
|
1018
|
-
Parameters
|
1019
|
-
----------
|
1020
|
-
type : str, default 'default'
|
1021
|
-
Card type.
|
1022
|
-
id : str, optional, default None
|
1023
|
-
If multiple cards are present, use this id to identify this card.
|
1024
|
-
options : Dict[str, Any], default {}
|
1025
|
-
Options passed to the card. The contents depend on the card type.
|
1026
|
-
timeout : int, default 45
|
1027
|
-
Interrupt reporting if it takes more than this many seconds.
|
1028
|
-
|
1029
|
-
|
1030
|
-
"""
|
1031
|
-
...
|
1032
|
-
|
1033
|
-
@typing.overload
|
1034
|
-
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1035
|
-
...
|
1036
|
-
|
1037
|
-
@typing.overload
|
1038
|
-
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1039
|
-
...
|
1040
|
-
|
1041
|
-
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
1042
|
-
"""
|
1043
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1044
|
-
|
1045
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1046
|
-
|
1047
|
-
Parameters
|
1048
|
-
----------
|
1049
|
-
type : str, default 'default'
|
1050
|
-
Card type.
|
1051
|
-
id : str, optional, default None
|
1052
|
-
If multiple cards are present, use this id to identify this card.
|
1053
|
-
options : Dict[str, Any], default {}
|
1054
|
-
Options passed to the card. The contents depend on the card type.
|
1055
|
-
timeout : int, default 45
|
1056
|
-
Interrupt reporting if it takes more than this many seconds.
|
1057
|
-
|
1058
|
-
|
1059
|
-
"""
|
1060
|
-
...
|
1061
|
-
|
1062
|
-
@typing.overload
|
1063
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1064
|
-
"""
|
1065
|
-
Specifies the PyPI packages for the step.
|
1066
|
-
|
1067
|
-
Information in this decorator will augment any
|
1068
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1069
|
-
you can use `@pypi_base` to set packages required by all
|
1070
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1071
|
-
|
1072
|
-
Parameters
|
1073
|
-
----------
|
1074
|
-
packages : Dict[str, str], default: {}
|
1075
|
-
Packages to use for this step. The key is the name of the package
|
1076
|
-
and the value is the version to use.
|
1077
|
-
python : str, optional, default: None
|
1078
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1079
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1080
|
-
"""
|
1081
|
-
...
|
1082
|
-
|
1083
|
-
@typing.overload
|
1084
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1085
|
-
...
|
1086
|
-
|
1087
|
-
@typing.overload
|
1088
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1089
|
-
...
|
1090
|
-
|
1091
|
-
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1092
|
-
"""
|
1093
|
-
Specifies the PyPI packages for the step.
|
1094
|
-
|
1095
|
-
Information in this decorator will augment any
|
1096
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1097
|
-
you can use `@pypi_base` to set packages required by all
|
1098
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1099
|
-
|
1100
|
-
Parameters
|
1101
|
-
----------
|
1102
|
-
packages : Dict[str, str], default: {}
|
1103
|
-
Packages to use for this step. The key is the name of the package
|
1104
|
-
and the value is the version to use.
|
1105
|
-
python : str, optional, default: None
|
1106
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1107
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1108
|
-
"""
|
1109
|
-
...
|
1110
|
-
|
1111
|
-
@typing.overload
|
1112
|
-
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1063
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1113
1064
|
"""
|
1114
|
-
Specifies
|
1115
|
-
|
1116
|
-
This decorator is useful if this step may hang indefinitely.
|
1117
|
-
|
1118
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1119
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1120
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1121
|
-
|
1122
|
-
Note that all the values specified in parameters are added together so if you specify
|
1123
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1065
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1124
1066
|
|
1125
1067
|
Parameters
|
1126
1068
|
----------
|
1127
|
-
|
1128
|
-
|
1129
|
-
minutes : int, default 0
|
1130
|
-
Number of minutes to wait prior to timing out.
|
1131
|
-
hours : int, default 0
|
1132
|
-
Number of hours to wait prior to timing out.
|
1069
|
+
vars : Dict[str, str], default {}
|
1070
|
+
Dictionary of environment variables to set.
|
1133
1071
|
"""
|
1134
1072
|
...
|
1135
1073
|
|
1136
1074
|
@typing.overload
|
1137
|
-
def
|
1075
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1138
1076
|
...
|
1139
1077
|
|
1140
1078
|
@typing.overload
|
1141
|
-
def
|
1079
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1142
1080
|
...
|
1143
1081
|
|
1144
|
-
def
|
1082
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1145
1083
|
"""
|
1146
|
-
Specifies
|
1147
|
-
|
1148
|
-
This decorator is useful if this step may hang indefinitely.
|
1149
|
-
|
1150
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1151
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1152
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1153
|
-
|
1154
|
-
Note that all the values specified in parameters are added together so if you specify
|
1155
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1084
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1156
1085
|
|
1157
1086
|
Parameters
|
1158
1087
|
----------
|
1159
|
-
|
1160
|
-
|
1161
|
-
minutes : int, default 0
|
1162
|
-
Number of minutes to wait prior to timing out.
|
1163
|
-
hours : int, default 0
|
1164
|
-
Number of hours to wait prior to timing out.
|
1088
|
+
vars : Dict[str, str], default {}
|
1089
|
+
Dictionary of environment variables to set.
|
1165
1090
|
"""
|
1166
1091
|
...
|
1167
1092
|
|
@@ -1215,42 +1140,95 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1215
1140
|
...
|
1216
1141
|
|
1217
1142
|
@typing.overload
|
1218
|
-
def
|
1143
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1219
1144
|
"""
|
1220
|
-
Specifies
|
1221
|
-
|
1145
|
+
Specifies the number of times the task corresponding
|
1146
|
+
to a step needs to be retried.
|
1147
|
+
|
1148
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
1149
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
1150
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
1151
|
+
|
1152
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
1153
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
1154
|
+
ensuring that the flow execution can continue.
|
1222
1155
|
|
1223
1156
|
Parameters
|
1224
1157
|
----------
|
1225
|
-
|
1226
|
-
|
1158
|
+
times : int, default 3
|
1159
|
+
Number of times to retry this task.
|
1160
|
+
minutes_between_retries : int, default 2
|
1161
|
+
Number of minutes between retries.
|
1227
1162
|
"""
|
1228
1163
|
...
|
1229
1164
|
|
1230
1165
|
@typing.overload
|
1231
|
-
def
|
1166
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1232
1167
|
...
|
1233
1168
|
|
1234
1169
|
@typing.overload
|
1235
|
-
def
|
1170
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1236
1171
|
...
|
1237
1172
|
|
1238
|
-
def
|
1173
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
1239
1174
|
"""
|
1240
|
-
Specifies
|
1241
|
-
|
1175
|
+
Specifies the number of times the task corresponding
|
1176
|
+
to a step needs to be retried.
|
1177
|
+
|
1178
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
1179
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
1180
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
1181
|
+
|
1182
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
1183
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
1184
|
+
ensuring that the flow execution can continue.
|
1242
1185
|
|
1243
1186
|
Parameters
|
1244
1187
|
----------
|
1245
|
-
|
1246
|
-
|
1188
|
+
times : int, default 3
|
1189
|
+
Number of times to retry this task.
|
1190
|
+
minutes_between_retries : int, default 2
|
1191
|
+
Number of minutes between retries.
|
1247
1192
|
"""
|
1248
1193
|
...
|
1249
1194
|
|
1250
1195
|
@typing.overload
|
1251
|
-
def
|
1196
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1252
1197
|
"""
|
1253
|
-
Specifies
|
1198
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1199
|
+
the execution of a step.
|
1200
|
+
|
1201
|
+
Parameters
|
1202
|
+
----------
|
1203
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1204
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1205
|
+
"""
|
1206
|
+
...
|
1207
|
+
|
1208
|
+
@typing.overload
|
1209
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1210
|
+
...
|
1211
|
+
|
1212
|
+
@typing.overload
|
1213
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1214
|
+
...
|
1215
|
+
|
1216
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
1217
|
+
"""
|
1218
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1219
|
+
the execution of a step.
|
1220
|
+
|
1221
|
+
Parameters
|
1222
|
+
----------
|
1223
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1224
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1225
|
+
"""
|
1226
|
+
...
|
1227
|
+
|
1228
|
+
@typing.overload
|
1229
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1230
|
+
"""
|
1231
|
+
Specifies the Conda environment for the step.
|
1254
1232
|
|
1255
1233
|
Information in this decorator will augment any
|
1256
1234
|
attributes set in the `@conda_base` flow-level decorator. Hence,
|
@@ -1305,230 +1283,108 @@ def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1305
1283
|
...
|
1306
1284
|
|
1307
1285
|
@typing.overload
|
1308
|
-
def
|
1309
|
-
"""
|
1310
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1311
|
-
|
1312
|
-
Parameters
|
1313
|
-
----------
|
1314
|
-
vars : Dict[str, str], default {}
|
1315
|
-
Dictionary of environment variables to set.
|
1316
|
-
"""
|
1317
|
-
...
|
1318
|
-
|
1319
|
-
@typing.overload
|
1320
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1321
|
-
...
|
1322
|
-
|
1323
|
-
@typing.overload
|
1324
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1325
|
-
...
|
1326
|
-
|
1327
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1286
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1328
1287
|
"""
|
1329
|
-
Specifies
|
1288
|
+
Specifies a timeout for your step.
|
1330
1289
|
|
1331
|
-
|
1332
|
-
----------
|
1333
|
-
vars : Dict[str, str], default {}
|
1334
|
-
Dictionary of environment variables to set.
|
1335
|
-
"""
|
1336
|
-
...
|
1337
|
-
|
1338
|
-
@typing.overload
|
1339
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1340
|
-
"""
|
1341
|
-
Specifies the number of times the task corresponding
|
1342
|
-
to a step needs to be retried.
|
1290
|
+
This decorator is useful if this step may hang indefinitely.
|
1343
1291
|
|
1344
|
-
This
|
1345
|
-
|
1346
|
-
|
1292
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1293
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1294
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1347
1295
|
|
1348
|
-
|
1349
|
-
decorator will
|
1350
|
-
ensuring that the flow execution can continue.
|
1296
|
+
Note that all the values specified in parameters are added together so if you specify
|
1297
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1351
1298
|
|
1352
1299
|
Parameters
|
1353
1300
|
----------
|
1354
|
-
|
1355
|
-
Number of
|
1356
|
-
|
1357
|
-
Number of minutes
|
1301
|
+
seconds : int, default 0
|
1302
|
+
Number of seconds to wait prior to timing out.
|
1303
|
+
minutes : int, default 0
|
1304
|
+
Number of minutes to wait prior to timing out.
|
1305
|
+
hours : int, default 0
|
1306
|
+
Number of hours to wait prior to timing out.
|
1358
1307
|
"""
|
1359
1308
|
...
|
1360
1309
|
|
1361
1310
|
@typing.overload
|
1362
|
-
def
|
1311
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1363
1312
|
...
|
1364
1313
|
|
1365
1314
|
@typing.overload
|
1366
|
-
def
|
1315
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1367
1316
|
...
|
1368
1317
|
|
1369
|
-
def
|
1318
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1370
1319
|
"""
|
1371
|
-
Specifies
|
1372
|
-
to a step needs to be retried.
|
1373
|
-
|
1374
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
1375
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
1376
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
1320
|
+
Specifies a timeout for your step.
|
1377
1321
|
|
1378
|
-
This
|
1379
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
1380
|
-
ensuring that the flow execution can continue.
|
1322
|
+
This decorator is useful if this step may hang indefinitely.
|
1381
1323
|
|
1382
|
-
|
1383
|
-
|
1384
|
-
|
1385
|
-
Number of times to retry this task.
|
1386
|
-
minutes_between_retries : int, default 2
|
1387
|
-
Number of minutes between retries.
|
1388
|
-
"""
|
1389
|
-
...
|
1390
|
-
|
1391
|
-
@typing.overload
|
1392
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1393
|
-
"""
|
1394
|
-
Specifies the Conda environment for all steps of the flow.
|
1324
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1325
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1326
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1395
1327
|
|
1396
|
-
|
1397
|
-
|
1328
|
+
Note that all the values specified in parameters are added together so if you specify
|
1329
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1398
1330
|
|
1399
1331
|
Parameters
|
1400
1332
|
----------
|
1401
|
-
|
1402
|
-
|
1403
|
-
|
1404
|
-
|
1405
|
-
|
1406
|
-
|
1407
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1408
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1409
|
-
disabled : bool, default False
|
1410
|
-
If set to True, disables Conda.
|
1333
|
+
seconds : int, default 0
|
1334
|
+
Number of seconds to wait prior to timing out.
|
1335
|
+
minutes : int, default 0
|
1336
|
+
Number of minutes to wait prior to timing out.
|
1337
|
+
hours : int, default 0
|
1338
|
+
Number of hours to wait prior to timing out.
|
1411
1339
|
"""
|
1412
1340
|
...
|
1413
1341
|
|
1414
1342
|
@typing.overload
|
1415
|
-
def
|
1416
|
-
...
|
1417
|
-
|
1418
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1343
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1419
1344
|
"""
|
1420
|
-
Specifies the
|
1345
|
+
Specifies the PyPI packages for the step.
|
1421
1346
|
|
1422
|
-
|
1423
|
-
|
1347
|
+
Information in this decorator will augment any
|
1348
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1349
|
+
you can use `@pypi_base` to set packages required by all
|
1350
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1424
1351
|
|
1425
1352
|
Parameters
|
1426
1353
|
----------
|
1427
|
-
packages : Dict[str, str], default {}
|
1428
|
-
Packages to use for this
|
1354
|
+
packages : Dict[str, str], default: {}
|
1355
|
+
Packages to use for this step. The key is the name of the package
|
1429
1356
|
and the value is the version to use.
|
1430
|
-
|
1431
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1432
|
-
python : str, optional, default None
|
1357
|
+
python : str, optional, default: None
|
1433
1358
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1434
1359
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1435
|
-
disabled : bool, default False
|
1436
|
-
If set to True, disables Conda.
|
1437
1360
|
"""
|
1438
1361
|
...
|
1439
1362
|
|
1440
1363
|
@typing.overload
|
1441
|
-
def
|
1442
|
-
"""
|
1443
|
-
Specifies the event(s) that this flow depends on.
|
1444
|
-
|
1445
|
-
```
|
1446
|
-
@trigger(event='foo')
|
1447
|
-
```
|
1448
|
-
or
|
1449
|
-
```
|
1450
|
-
@trigger(events=['foo', 'bar'])
|
1451
|
-
```
|
1452
|
-
|
1453
|
-
Additionally, you can specify the parameter mappings
|
1454
|
-
to map event payload to Metaflow parameters for the flow.
|
1455
|
-
```
|
1456
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1457
|
-
```
|
1458
|
-
or
|
1459
|
-
```
|
1460
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1461
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1462
|
-
```
|
1463
|
-
|
1464
|
-
'parameters' can also be a list of strings and tuples like so:
|
1465
|
-
```
|
1466
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1467
|
-
```
|
1468
|
-
This is equivalent to:
|
1469
|
-
```
|
1470
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1471
|
-
```
|
1472
|
-
|
1473
|
-
Parameters
|
1474
|
-
----------
|
1475
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
1476
|
-
Event dependency for this flow.
|
1477
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
1478
|
-
Events dependency for this flow.
|
1479
|
-
options : Dict[str, Any], default {}
|
1480
|
-
Backend-specific configuration for tuning eventing behavior.
|
1481
|
-
|
1482
|
-
|
1483
|
-
"""
|
1364
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1484
1365
|
...
|
1485
1366
|
|
1486
1367
|
@typing.overload
|
1487
|
-
def
|
1368
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1488
1369
|
...
|
1489
1370
|
|
1490
|
-
def
|
1371
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1491
1372
|
"""
|
1492
|
-
Specifies the
|
1493
|
-
|
1494
|
-
```
|
1495
|
-
@trigger(event='foo')
|
1496
|
-
```
|
1497
|
-
or
|
1498
|
-
```
|
1499
|
-
@trigger(events=['foo', 'bar'])
|
1500
|
-
```
|
1501
|
-
|
1502
|
-
Additionally, you can specify the parameter mappings
|
1503
|
-
to map event payload to Metaflow parameters for the flow.
|
1504
|
-
```
|
1505
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1506
|
-
```
|
1507
|
-
or
|
1508
|
-
```
|
1509
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1510
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1511
|
-
```
|
1373
|
+
Specifies the PyPI packages for the step.
|
1512
1374
|
|
1513
|
-
|
1514
|
-
|
1515
|
-
|
1516
|
-
|
1517
|
-
This is equivalent to:
|
1518
|
-
```
|
1519
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1520
|
-
```
|
1375
|
+
Information in this decorator will augment any
|
1376
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1377
|
+
you can use `@pypi_base` to set packages required by all
|
1378
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1521
1379
|
|
1522
1380
|
Parameters
|
1523
1381
|
----------
|
1524
|
-
|
1525
|
-
|
1526
|
-
|
1527
|
-
|
1528
|
-
|
1529
|
-
|
1530
|
-
|
1531
|
-
|
1382
|
+
packages : Dict[str, str], default: {}
|
1383
|
+
Packages to use for this step. The key is the name of the package
|
1384
|
+
and the value is the version to use.
|
1385
|
+
python : str, optional, default: None
|
1386
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1387
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1532
1388
|
"""
|
1533
1389
|
...
|
1534
1390
|
|
@@ -1613,6 +1469,55 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1613
1469
|
"""
|
1614
1470
|
...
|
1615
1471
|
|
1472
|
+
@typing.overload
|
1473
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1474
|
+
"""
|
1475
|
+
Specifies the Conda environment for all steps of the flow.
|
1476
|
+
|
1477
|
+
Use `@conda_base` to set common libraries required by all
|
1478
|
+
steps and use `@conda` to specify step-specific additions.
|
1479
|
+
|
1480
|
+
Parameters
|
1481
|
+
----------
|
1482
|
+
packages : Dict[str, str], default {}
|
1483
|
+
Packages to use for this flow. The key is the name of the package
|
1484
|
+
and the value is the version to use.
|
1485
|
+
libraries : Dict[str, str], default {}
|
1486
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1487
|
+
python : str, optional, default None
|
1488
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1489
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1490
|
+
disabled : bool, default False
|
1491
|
+
If set to True, disables Conda.
|
1492
|
+
"""
|
1493
|
+
...
|
1494
|
+
|
1495
|
+
@typing.overload
|
1496
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1497
|
+
...
|
1498
|
+
|
1499
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1500
|
+
"""
|
1501
|
+
Specifies the Conda environment for all steps of the flow.
|
1502
|
+
|
1503
|
+
Use `@conda_base` to set common libraries required by all
|
1504
|
+
steps and use `@conda` to specify step-specific additions.
|
1505
|
+
|
1506
|
+
Parameters
|
1507
|
+
----------
|
1508
|
+
packages : Dict[str, str], default {}
|
1509
|
+
Packages to use for this flow. The key is the name of the package
|
1510
|
+
and the value is the version to use.
|
1511
|
+
libraries : Dict[str, str], default {}
|
1512
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1513
|
+
python : str, optional, default None
|
1514
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1515
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1516
|
+
disabled : bool, default False
|
1517
|
+
If set to True, disables Conda.
|
1518
|
+
"""
|
1519
|
+
...
|
1520
|
+
|
1616
1521
|
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1617
1522
|
"""
|
1618
1523
|
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
@@ -1656,67 +1561,95 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1656
1561
|
...
|
1657
1562
|
|
1658
1563
|
@typing.overload
|
1659
|
-
def
|
1564
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1660
1565
|
"""
|
1661
|
-
Specifies the
|
1662
|
-
|
1566
|
+
Specifies the event(s) that this flow depends on.
|
1567
|
+
|
1568
|
+
```
|
1569
|
+
@trigger(event='foo')
|
1570
|
+
```
|
1571
|
+
or
|
1572
|
+
```
|
1573
|
+
@trigger(events=['foo', 'bar'])
|
1574
|
+
```
|
1575
|
+
|
1576
|
+
Additionally, you can specify the parameter mappings
|
1577
|
+
to map event payload to Metaflow parameters for the flow.
|
1578
|
+
```
|
1579
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1580
|
+
```
|
1581
|
+
or
|
1582
|
+
```
|
1583
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1584
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1585
|
+
```
|
1586
|
+
|
1587
|
+
'parameters' can also be a list of strings and tuples like so:
|
1588
|
+
```
|
1589
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1590
|
+
```
|
1591
|
+
This is equivalent to:
|
1592
|
+
```
|
1593
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1594
|
+
```
|
1663
1595
|
|
1664
1596
|
Parameters
|
1665
1597
|
----------
|
1666
|
-
|
1667
|
-
|
1668
|
-
|
1669
|
-
|
1670
|
-
|
1671
|
-
|
1672
|
-
|
1673
|
-
|
1674
|
-
specified by this expression.
|
1675
|
-
timezone : str, optional, default None
|
1676
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1677
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1598
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1599
|
+
Event dependency for this flow.
|
1600
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1601
|
+
Events dependency for this flow.
|
1602
|
+
options : Dict[str, Any], default {}
|
1603
|
+
Backend-specific configuration for tuning eventing behavior.
|
1604
|
+
|
1605
|
+
|
1678
1606
|
"""
|
1679
1607
|
...
|
1680
1608
|
|
1681
1609
|
@typing.overload
|
1682
|
-
def
|
1610
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1683
1611
|
...
|
1684
1612
|
|
1685
|
-
def
|
1613
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1686
1614
|
"""
|
1687
|
-
Specifies the
|
1688
|
-
production scheduler.
|
1615
|
+
Specifies the event(s) that this flow depends on.
|
1689
1616
|
|
1690
|
-
|
1691
|
-
|
1692
|
-
|
1693
|
-
|
1694
|
-
|
1695
|
-
|
1696
|
-
|
1697
|
-
Run the workflow weekly.
|
1698
|
-
cron : str, optional, default None
|
1699
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1700
|
-
specified by this expression.
|
1701
|
-
timezone : str, optional, default None
|
1702
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1703
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1704
|
-
"""
|
1705
|
-
...
|
1706
|
-
|
1707
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1708
|
-
"""
|
1709
|
-
Specifies what flows belong to the same project.
|
1617
|
+
```
|
1618
|
+
@trigger(event='foo')
|
1619
|
+
```
|
1620
|
+
or
|
1621
|
+
```
|
1622
|
+
@trigger(events=['foo', 'bar'])
|
1623
|
+
```
|
1710
1624
|
|
1711
|
-
|
1712
|
-
|
1625
|
+
Additionally, you can specify the parameter mappings
|
1626
|
+
to map event payload to Metaflow parameters for the flow.
|
1627
|
+
```
|
1628
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1629
|
+
```
|
1630
|
+
or
|
1631
|
+
```
|
1632
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1633
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1634
|
+
```
|
1635
|
+
|
1636
|
+
'parameters' can also be a list of strings and tuples like so:
|
1637
|
+
```
|
1638
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1639
|
+
```
|
1640
|
+
This is equivalent to:
|
1641
|
+
```
|
1642
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1643
|
+
```
|
1713
1644
|
|
1714
1645
|
Parameters
|
1715
1646
|
----------
|
1716
|
-
|
1717
|
-
|
1718
|
-
|
1719
|
-
|
1647
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1648
|
+
Event dependency for this flow.
|
1649
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1650
|
+
Events dependency for this flow.
|
1651
|
+
options : Dict[str, Any], default {}
|
1652
|
+
Backend-specific configuration for tuning eventing behavior.
|
1720
1653
|
|
1721
1654
|
|
1722
1655
|
"""
|
@@ -1825,6 +1758,73 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1825
1758
|
"""
|
1826
1759
|
...
|
1827
1760
|
|
1761
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1762
|
+
"""
|
1763
|
+
Specifies what flows belong to the same project.
|
1764
|
+
|
1765
|
+
A project-specific namespace is created for all flows that
|
1766
|
+
use the same `@project(name)`.
|
1767
|
+
|
1768
|
+
Parameters
|
1769
|
+
----------
|
1770
|
+
name : str
|
1771
|
+
Project name. Make sure that the name is unique amongst all
|
1772
|
+
projects that use the same production scheduler. The name may
|
1773
|
+
contain only lowercase alphanumeric characters and underscores.
|
1774
|
+
|
1775
|
+
|
1776
|
+
"""
|
1777
|
+
...
|
1778
|
+
|
1779
|
+
@typing.overload
|
1780
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1781
|
+
"""
|
1782
|
+
Specifies the times when the flow should be run when running on a
|
1783
|
+
production scheduler.
|
1784
|
+
|
1785
|
+
Parameters
|
1786
|
+
----------
|
1787
|
+
hourly : bool, default False
|
1788
|
+
Run the workflow hourly.
|
1789
|
+
daily : bool, default True
|
1790
|
+
Run the workflow daily.
|
1791
|
+
weekly : bool, default False
|
1792
|
+
Run the workflow weekly.
|
1793
|
+
cron : str, optional, default None
|
1794
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1795
|
+
specified by this expression.
|
1796
|
+
timezone : str, optional, default None
|
1797
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1798
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1799
|
+
"""
|
1800
|
+
...
|
1801
|
+
|
1802
|
+
@typing.overload
|
1803
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1804
|
+
...
|
1805
|
+
|
1806
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1807
|
+
"""
|
1808
|
+
Specifies the times when the flow should be run when running on a
|
1809
|
+
production scheduler.
|
1810
|
+
|
1811
|
+
Parameters
|
1812
|
+
----------
|
1813
|
+
hourly : bool, default False
|
1814
|
+
Run the workflow hourly.
|
1815
|
+
daily : bool, default True
|
1816
|
+
Run the workflow daily.
|
1817
|
+
weekly : bool, default False
|
1818
|
+
Run the workflow weekly.
|
1819
|
+
cron : str, optional, default None
|
1820
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1821
|
+
specified by this expression.
|
1822
|
+
timezone : str, optional, default None
|
1823
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1824
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1825
|
+
"""
|
1826
|
+
...
|
1827
|
+
|
1828
1828
|
def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
1829
1829
|
"""
|
1830
1830
|
Switch namespace to the one provided.
|