metaflow-stubs 2.12.5__py2.py3-none-any.whl → 2.12.7__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +386 -386
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +3 -3
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +4 -4
- metaflow-stubs/metadata/metadata.pyi +3 -3
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +17 -17
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +6 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +7 -7
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +4 -4
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +112 -0
- metaflow-stubs/system/system_logger.pyi +51 -0
- metaflow-stubs/system/system_monitor.pyi +73 -0
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.7.dist-info}/METADATA +2 -2
- metaflow_stubs-2.12.7.dist-info/RECORD +145 -0
- {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.7.dist-info}/WHEEL +1 -1
- metaflow_stubs-2.12.5.dist-info/RECORD +0 -142
- {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.7.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.7 #
|
4
|
+
# Generated on 2024-07-03T19:20:47.569890 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
@@ -9,17 +9,17 @@ from __future__ import annotations
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
11
|
import metaflow.client.core
|
12
|
-
import metaflow.
|
12
|
+
import metaflow.datastore.inputs
|
13
|
+
import metaflow.parameters
|
14
|
+
import metaflow.metaflow_current
|
13
15
|
import metaflow._vendor.click.types
|
14
16
|
import typing
|
15
|
-
import metaflow.
|
16
|
-
import
|
17
|
+
import metaflow.runner.metaflow_runner
|
18
|
+
import io
|
17
19
|
import metaflow.plugins.datatools.s3.s3
|
18
20
|
import datetime
|
19
|
-
import io
|
20
|
-
import metaflow.metaflow_current
|
21
|
-
import metaflow.datastore.inputs
|
22
21
|
import metaflow.events
|
22
|
+
import metaflow.flowspec
|
23
23
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
24
24
|
StepFlag = typing.NewType("StepFlag", bool)
|
25
25
|
|
@@ -728,35 +728,110 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
728
728
|
...
|
729
729
|
|
730
730
|
@typing.overload
|
731
|
-
def
|
731
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
732
732
|
"""
|
733
|
-
Specifies
|
734
|
-
|
733
|
+
Specifies the Conda environment for the step.
|
734
|
+
|
735
|
+
Information in this decorator will augment any
|
736
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
737
|
+
you can use `@conda_base` to set packages required by all
|
738
|
+
steps and use `@conda` to specify step-specific overrides.
|
735
739
|
|
736
740
|
Parameters
|
737
741
|
----------
|
738
|
-
|
739
|
-
|
742
|
+
packages : Dict[str, str], default {}
|
743
|
+
Packages to use for this step. The key is the name of the package
|
744
|
+
and the value is the version to use.
|
745
|
+
libraries : Dict[str, str], default {}
|
746
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
747
|
+
python : str, optional, default None
|
748
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
749
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
750
|
+
disabled : bool, default False
|
751
|
+
If set to True, disables @conda.
|
740
752
|
"""
|
741
753
|
...
|
742
754
|
|
743
755
|
@typing.overload
|
744
|
-
def
|
756
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
745
757
|
...
|
746
758
|
|
747
759
|
@typing.overload
|
748
|
-
def
|
760
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
749
761
|
...
|
750
762
|
|
751
|
-
def
|
763
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
752
764
|
"""
|
753
|
-
Specifies
|
754
|
-
|
765
|
+
Specifies the Conda environment for the step.
|
766
|
+
|
767
|
+
Information in this decorator will augment any
|
768
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
769
|
+
you can use `@conda_base` to set packages required by all
|
770
|
+
steps and use `@conda` to specify step-specific overrides.
|
755
771
|
|
756
772
|
Parameters
|
757
773
|
----------
|
758
|
-
|
759
|
-
|
774
|
+
packages : Dict[str, str], default {}
|
775
|
+
Packages to use for this step. The key is the name of the package
|
776
|
+
and the value is the version to use.
|
777
|
+
libraries : Dict[str, str], default {}
|
778
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
779
|
+
python : str, optional, default None
|
780
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
781
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
782
|
+
disabled : bool, default False
|
783
|
+
If set to True, disables @conda.
|
784
|
+
"""
|
785
|
+
...
|
786
|
+
|
787
|
+
@typing.overload
|
788
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
789
|
+
"""
|
790
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
791
|
+
|
792
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
793
|
+
|
794
|
+
Parameters
|
795
|
+
----------
|
796
|
+
type : str, default 'default'
|
797
|
+
Card type.
|
798
|
+
id : str, optional, default None
|
799
|
+
If multiple cards are present, use this id to identify this card.
|
800
|
+
options : Dict[str, Any], default {}
|
801
|
+
Options passed to the card. The contents depend on the card type.
|
802
|
+
timeout : int, default 45
|
803
|
+
Interrupt reporting if it takes more than this many seconds.
|
804
|
+
|
805
|
+
|
806
|
+
"""
|
807
|
+
...
|
808
|
+
|
809
|
+
@typing.overload
|
810
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
811
|
+
...
|
812
|
+
|
813
|
+
@typing.overload
|
814
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
815
|
+
...
|
816
|
+
|
817
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
818
|
+
"""
|
819
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
820
|
+
|
821
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
822
|
+
|
823
|
+
Parameters
|
824
|
+
----------
|
825
|
+
type : str, default 'default'
|
826
|
+
Card type.
|
827
|
+
id : str, optional, default None
|
828
|
+
If multiple cards are present, use this id to identify this card.
|
829
|
+
options : Dict[str, Any], default {}
|
830
|
+
Options passed to the card. The contents depend on the card type.
|
831
|
+
timeout : int, default 45
|
832
|
+
Interrupt reporting if it takes more than this many seconds.
|
833
|
+
|
834
|
+
|
760
835
|
"""
|
761
836
|
...
|
762
837
|
|
@@ -907,201 +982,101 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
907
982
|
"""
|
908
983
|
...
|
909
984
|
|
910
|
-
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
911
|
-
"""
|
912
|
-
Specifies that this step should execute on Kubernetes.
|
913
|
-
|
914
|
-
Parameters
|
915
|
-
----------
|
916
|
-
cpu : int, default 1
|
917
|
-
Number of CPUs required for this step. If `@resources` is
|
918
|
-
also present, the maximum value from all decorators is used.
|
919
|
-
memory : int, default 4096
|
920
|
-
Memory size (in MB) required for this step. If
|
921
|
-
`@resources` is also present, the maximum value from all decorators is
|
922
|
-
used.
|
923
|
-
disk : int, default 10240
|
924
|
-
Disk size (in MB) required for this step. If
|
925
|
-
`@resources` is also present, the maximum value from all decorators is
|
926
|
-
used.
|
927
|
-
image : str, optional, default None
|
928
|
-
Docker image to use when launching on Kubernetes. If not specified, and
|
929
|
-
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
930
|
-
not, a default Docker image mapping to the current version of Python is used.
|
931
|
-
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
932
|
-
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
933
|
-
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
934
|
-
Kubernetes service account to use when launching pod in Kubernetes.
|
935
|
-
secrets : List[str], optional, default None
|
936
|
-
Kubernetes secrets to use when launching pod in Kubernetes. These
|
937
|
-
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
938
|
-
in Metaflow configuration.
|
939
|
-
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
940
|
-
Kubernetes namespace to use when launching pod in Kubernetes.
|
941
|
-
gpu : int, optional, default None
|
942
|
-
Number of GPUs required for this step. A value of zero implies that
|
943
|
-
the scheduled node should not have GPUs.
|
944
|
-
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
945
|
-
The vendor of the GPUs to be used for this step.
|
946
|
-
tolerations : List[str], default []
|
947
|
-
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
948
|
-
Kubernetes tolerations to use when launching pod in Kubernetes.
|
949
|
-
use_tmpfs : bool, default False
|
950
|
-
This enables an explicit tmpfs mount for this step.
|
951
|
-
tmpfs_tempdir : bool, default True
|
952
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
953
|
-
tmpfs_size : int, optional, default: None
|
954
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
955
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
956
|
-
memory allocated for this step.
|
957
|
-
tmpfs_path : str, optional, default /metaflow_temp
|
958
|
-
Path to tmpfs mount for this step.
|
959
|
-
persistent_volume_claims : Dict[str, str], optional, default None
|
960
|
-
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
961
|
-
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
962
|
-
shared_memory: int, optional
|
963
|
-
Shared memory size (in MiB) required for this step
|
964
|
-
port: int, optional
|
965
|
-
Port number to specify in the Kubernetes job object
|
966
|
-
"""
|
967
|
-
...
|
968
|
-
|
969
985
|
@typing.overload
|
970
|
-
def
|
986
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
971
987
|
"""
|
972
|
-
Specifies the
|
988
|
+
Specifies the PyPI packages for the step.
|
973
989
|
|
974
990
|
Information in this decorator will augment any
|
975
|
-
attributes set in the `@
|
976
|
-
you can use `@
|
977
|
-
steps and use `@
|
991
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
992
|
+
you can use `@pypi_base` to set packages required by all
|
993
|
+
steps and use `@pypi` to specify step-specific overrides.
|
978
994
|
|
979
995
|
Parameters
|
980
996
|
----------
|
981
|
-
packages : Dict[str, str], default {}
|
997
|
+
packages : Dict[str, str], default: {}
|
982
998
|
Packages to use for this step. The key is the name of the package
|
983
999
|
and the value is the version to use.
|
984
|
-
|
985
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
986
|
-
python : str, optional, default None
|
1000
|
+
python : str, optional, default: None
|
987
1001
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
988
1002
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
989
|
-
disabled : bool, default False
|
990
|
-
If set to True, disables @conda.
|
991
1003
|
"""
|
992
1004
|
...
|
993
1005
|
|
994
1006
|
@typing.overload
|
995
|
-
def
|
1007
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
996
1008
|
...
|
997
1009
|
|
998
1010
|
@typing.overload
|
999
|
-
def
|
1011
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1000
1012
|
...
|
1001
1013
|
|
1002
|
-
def
|
1014
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1003
1015
|
"""
|
1004
|
-
Specifies the
|
1016
|
+
Specifies the PyPI packages for the step.
|
1005
1017
|
|
1006
1018
|
Information in this decorator will augment any
|
1007
|
-
attributes set in the `@
|
1008
|
-
you can use `@
|
1009
|
-
steps and use `@
|
1019
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1020
|
+
you can use `@pypi_base` to set packages required by all
|
1021
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1010
1022
|
|
1011
1023
|
Parameters
|
1012
1024
|
----------
|
1013
|
-
packages : Dict[str, str], default {}
|
1025
|
+
packages : Dict[str, str], default: {}
|
1014
1026
|
Packages to use for this step. The key is the name of the package
|
1015
1027
|
and the value is the version to use.
|
1016
|
-
|
1017
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1018
|
-
python : str, optional, default None
|
1028
|
+
python : str, optional, default: None
|
1019
1029
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1020
1030
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1021
|
-
disabled : bool, default False
|
1022
|
-
If set to True, disables @conda.
|
1023
1031
|
"""
|
1024
1032
|
...
|
1025
1033
|
|
1026
1034
|
@typing.overload
|
1027
|
-
def
|
1035
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1028
1036
|
"""
|
1029
|
-
|
1037
|
+
Specifies that the step will success under all circumstances.
|
1030
1038
|
|
1031
|
-
|
1039
|
+
The decorator will create an optional artifact, specified by `var`, which
|
1040
|
+
contains the exception raised. You can use it to detect the presence
|
1041
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
1042
|
+
are missing.
|
1032
1043
|
|
1033
1044
|
Parameters
|
1034
1045
|
----------
|
1035
|
-
|
1036
|
-
|
1037
|
-
|
1038
|
-
|
1039
|
-
|
1040
|
-
|
1041
|
-
timeout : int, default 45
|
1042
|
-
Interrupt reporting if it takes more than this many seconds.
|
1043
|
-
|
1044
|
-
|
1046
|
+
var : str, optional, default None
|
1047
|
+
Name of the artifact in which to store the caught exception.
|
1048
|
+
If not specified, the exception is not stored.
|
1049
|
+
print_exception : bool, default True
|
1050
|
+
Determines whether or not the exception is printed to
|
1051
|
+
stdout when caught.
|
1045
1052
|
"""
|
1046
1053
|
...
|
1047
1054
|
|
1048
1055
|
@typing.overload
|
1049
|
-
def
|
1056
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1050
1057
|
...
|
1051
1058
|
|
1052
1059
|
@typing.overload
|
1053
|
-
def
|
1060
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1054
1061
|
...
|
1055
1062
|
|
1056
|
-
def
|
1063
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
1057
1064
|
"""
|
1058
|
-
|
1065
|
+
Specifies that the step will success under all circumstances.
|
1059
1066
|
|
1060
|
-
|
1067
|
+
The decorator will create an optional artifact, specified by `var`, which
|
1068
|
+
contains the exception raised. You can use it to detect the presence
|
1069
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
1070
|
+
are missing.
|
1061
1071
|
|
1062
1072
|
Parameters
|
1063
1073
|
----------
|
1064
|
-
|
1065
|
-
|
1066
|
-
|
1067
|
-
|
1068
|
-
|
1069
|
-
|
1070
|
-
timeout : int, default 45
|
1071
|
-
Interrupt reporting if it takes more than this many seconds.
|
1072
|
-
|
1073
|
-
|
1074
|
-
"""
|
1075
|
-
...
|
1076
|
-
|
1077
|
-
@typing.overload
|
1078
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1079
|
-
"""
|
1080
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1081
|
-
|
1082
|
-
Parameters
|
1083
|
-
----------
|
1084
|
-
vars : Dict[str, str], default {}
|
1085
|
-
Dictionary of environment variables to set.
|
1086
|
-
"""
|
1087
|
-
...
|
1088
|
-
|
1089
|
-
@typing.overload
|
1090
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1091
|
-
...
|
1092
|
-
|
1093
|
-
@typing.overload
|
1094
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1095
|
-
...
|
1096
|
-
|
1097
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1098
|
-
"""
|
1099
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1100
|
-
|
1101
|
-
Parameters
|
1102
|
-
----------
|
1103
|
-
vars : Dict[str, str], default {}
|
1104
|
-
Dictionary of environment variables to set.
|
1074
|
+
var : str, optional, default None
|
1075
|
+
Name of the artifact in which to store the caught exception.
|
1076
|
+
If not specified, the exception is not stored.
|
1077
|
+
print_exception : bool, default True
|
1078
|
+
Determines whether or not the exception is printed to
|
1079
|
+
stdout when caught.
|
1105
1080
|
"""
|
1106
1081
|
...
|
1107
1082
|
|
@@ -1159,51 +1134,151 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1159
1134
|
...
|
1160
1135
|
|
1161
1136
|
@typing.overload
|
1162
|
-
def
|
1137
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1163
1138
|
"""
|
1164
|
-
Specifies
|
1139
|
+
Specifies a timeout for your step.
|
1165
1140
|
|
1166
|
-
|
1167
|
-
|
1168
|
-
|
1169
|
-
|
1141
|
+
This decorator is useful if this step may hang indefinitely.
|
1142
|
+
|
1143
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1144
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1145
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1146
|
+
|
1147
|
+
Note that all the values specified in parameters are added together so if you specify
|
1148
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1170
1149
|
|
1171
1150
|
Parameters
|
1172
1151
|
----------
|
1173
|
-
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1152
|
+
seconds : int, default 0
|
1153
|
+
Number of seconds to wait prior to timing out.
|
1154
|
+
minutes : int, default 0
|
1155
|
+
Number of minutes to wait prior to timing out.
|
1156
|
+
hours : int, default 0
|
1157
|
+
Number of hours to wait prior to timing out.
|
1179
1158
|
"""
|
1180
1159
|
...
|
1181
1160
|
|
1182
1161
|
@typing.overload
|
1183
|
-
def
|
1162
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1184
1163
|
...
|
1185
1164
|
|
1186
1165
|
@typing.overload
|
1187
|
-
def
|
1166
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1188
1167
|
...
|
1189
1168
|
|
1190
|
-
def
|
1169
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1191
1170
|
"""
|
1192
|
-
Specifies
|
1171
|
+
Specifies a timeout for your step.
|
1193
1172
|
|
1194
|
-
|
1195
|
-
|
1196
|
-
|
1197
|
-
|
1173
|
+
This decorator is useful if this step may hang indefinitely.
|
1174
|
+
|
1175
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1176
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1177
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1178
|
+
|
1179
|
+
Note that all the values specified in parameters are added together so if you specify
|
1180
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1198
1181
|
|
1199
1182
|
Parameters
|
1200
1183
|
----------
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1184
|
+
seconds : int, default 0
|
1185
|
+
Number of seconds to wait prior to timing out.
|
1186
|
+
minutes : int, default 0
|
1187
|
+
Number of minutes to wait prior to timing out.
|
1188
|
+
hours : int, default 0
|
1189
|
+
Number of hours to wait prior to timing out.
|
1190
|
+
"""
|
1191
|
+
...
|
1192
|
+
|
1193
|
+
@typing.overload
|
1194
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1195
|
+
"""
|
1196
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1197
|
+
the execution of a step.
|
1198
|
+
|
1199
|
+
Parameters
|
1200
|
+
----------
|
1201
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1202
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1203
|
+
"""
|
1204
|
+
...
|
1205
|
+
|
1206
|
+
@typing.overload
|
1207
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1208
|
+
...
|
1209
|
+
|
1210
|
+
@typing.overload
|
1211
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1212
|
+
...
|
1213
|
+
|
1214
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
1215
|
+
"""
|
1216
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1217
|
+
the execution of a step.
|
1218
|
+
|
1219
|
+
Parameters
|
1220
|
+
----------
|
1221
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1222
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1223
|
+
"""
|
1224
|
+
...
|
1225
|
+
|
1226
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1227
|
+
"""
|
1228
|
+
Specifies that this step should execute on Kubernetes.
|
1229
|
+
|
1230
|
+
Parameters
|
1231
|
+
----------
|
1232
|
+
cpu : int, default 1
|
1233
|
+
Number of CPUs required for this step. If `@resources` is
|
1234
|
+
also present, the maximum value from all decorators is used.
|
1235
|
+
memory : int, default 4096
|
1236
|
+
Memory size (in MB) required for this step. If
|
1237
|
+
`@resources` is also present, the maximum value from all decorators is
|
1238
|
+
used.
|
1239
|
+
disk : int, default 10240
|
1240
|
+
Disk size (in MB) required for this step. If
|
1241
|
+
`@resources` is also present, the maximum value from all decorators is
|
1242
|
+
used.
|
1243
|
+
image : str, optional, default None
|
1244
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
1245
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
1246
|
+
not, a default Docker image mapping to the current version of Python is used.
|
1247
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
1248
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
1249
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
1250
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
1251
|
+
secrets : List[str], optional, default None
|
1252
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
1253
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
1254
|
+
in Metaflow configuration.
|
1255
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
1256
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
1257
|
+
gpu : int, optional, default None
|
1258
|
+
Number of GPUs required for this step. A value of zero implies that
|
1259
|
+
the scheduled node should not have GPUs.
|
1260
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
1261
|
+
The vendor of the GPUs to be used for this step.
|
1262
|
+
tolerations : List[str], default []
|
1263
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
1264
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
1265
|
+
use_tmpfs : bool, default False
|
1266
|
+
This enables an explicit tmpfs mount for this step.
|
1267
|
+
tmpfs_tempdir : bool, default True
|
1268
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
1269
|
+
tmpfs_size : int, optional, default: None
|
1270
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
1271
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
1272
|
+
memory allocated for this step.
|
1273
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
1274
|
+
Path to tmpfs mount for this step.
|
1275
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
1276
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
1277
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
1278
|
+
shared_memory: int, optional
|
1279
|
+
Shared memory size (in MiB) required for this step
|
1280
|
+
port: int, optional
|
1281
|
+
Port number to specify in the Kubernetes job object
|
1207
1282
|
"""
|
1208
1283
|
...
|
1209
1284
|
|
@@ -1285,157 +1360,93 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
1285
1360
|
...
|
1286
1361
|
|
1287
1362
|
@typing.overload
|
1288
|
-
def
|
1363
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1289
1364
|
"""
|
1290
|
-
Specifies
|
1291
|
-
|
1292
|
-
This decorator is useful if this step may hang indefinitely.
|
1293
|
-
|
1294
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1295
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1296
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1297
|
-
|
1298
|
-
Note that all the values specified in parameters are added together so if you specify
|
1299
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1365
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1300
1366
|
|
1301
1367
|
Parameters
|
1302
1368
|
----------
|
1303
|
-
|
1304
|
-
|
1305
|
-
minutes : int, default 0
|
1306
|
-
Number of minutes to wait prior to timing out.
|
1307
|
-
hours : int, default 0
|
1308
|
-
Number of hours to wait prior to timing out.
|
1369
|
+
vars : Dict[str, str], default {}
|
1370
|
+
Dictionary of environment variables to set.
|
1309
1371
|
"""
|
1310
1372
|
...
|
1311
1373
|
|
1312
1374
|
@typing.overload
|
1313
|
-
def
|
1375
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1314
1376
|
...
|
1315
1377
|
|
1316
1378
|
@typing.overload
|
1317
|
-
def
|
1379
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1318
1380
|
...
|
1319
1381
|
|
1320
|
-
def
|
1382
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1321
1383
|
"""
|
1322
|
-
Specifies
|
1323
|
-
|
1324
|
-
This decorator is useful if this step may hang indefinitely.
|
1325
|
-
|
1326
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1327
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1328
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1329
|
-
|
1330
|
-
Note that all the values specified in parameters are added together so if you specify
|
1331
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1384
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1332
1385
|
|
1333
1386
|
Parameters
|
1334
1387
|
----------
|
1335
|
-
|
1336
|
-
|
1337
|
-
minutes : int, default 0
|
1338
|
-
Number of minutes to wait prior to timing out.
|
1339
|
-
hours : int, default 0
|
1340
|
-
Number of hours to wait prior to timing out.
|
1388
|
+
vars : Dict[str, str], default {}
|
1389
|
+
Dictionary of environment variables to set.
|
1341
1390
|
"""
|
1342
1391
|
...
|
1343
1392
|
|
1344
|
-
|
1345
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1393
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1346
1394
|
"""
|
1347
|
-
|
1348
|
-
|
1349
|
-
Information in this decorator will augment any
|
1350
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1351
|
-
you can use `@pypi_base` to set packages required by all
|
1352
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1395
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1396
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1353
1397
|
|
1354
1398
|
Parameters
|
1355
1399
|
----------
|
1356
|
-
|
1357
|
-
|
1358
|
-
|
1359
|
-
|
1360
|
-
|
1361
|
-
|
1400
|
+
timeout : int
|
1401
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1402
|
+
poke_interval : int
|
1403
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1404
|
+
mode : str
|
1405
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1406
|
+
exponential_backoff : bool
|
1407
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1408
|
+
pool : str
|
1409
|
+
the slot pool this task should run in,
|
1410
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1411
|
+
soft_fail : bool
|
1412
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1413
|
+
name : str
|
1414
|
+
Name of the sensor on Airflow
|
1415
|
+
description : str
|
1416
|
+
Description of sensor in the Airflow UI
|
1417
|
+
external_dag_id : str
|
1418
|
+
The dag_id that contains the task you want to wait for.
|
1419
|
+
external_task_ids : List[str]
|
1420
|
+
The list of task_ids that you want to wait for.
|
1421
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1422
|
+
allowed_states : List[str]
|
1423
|
+
Iterable of allowed states, (Default: ['success'])
|
1424
|
+
failed_states : List[str]
|
1425
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1426
|
+
execution_delta : datetime.timedelta
|
1427
|
+
time difference with the previous execution to look at,
|
1428
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1429
|
+
check_existence: bool
|
1430
|
+
Set to True to check if the external task exists or check if
|
1431
|
+
the DAG to wait for exists. (Default: True)
|
1362
1432
|
"""
|
1363
1433
|
...
|
1364
1434
|
|
1365
|
-
|
1366
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1367
|
-
...
|
1368
|
-
|
1369
|
-
@typing.overload
|
1370
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1371
|
-
...
|
1372
|
-
|
1373
|
-
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1435
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1374
1436
|
"""
|
1375
|
-
Specifies
|
1437
|
+
Specifies what flows belong to the same project.
|
1376
1438
|
|
1377
|
-
|
1378
|
-
|
1379
|
-
you can use `@pypi_base` to set packages required by all
|
1380
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1439
|
+
A project-specific namespace is created for all flows that
|
1440
|
+
use the same `@project(name)`.
|
1381
1441
|
|
1382
1442
|
Parameters
|
1383
1443
|
----------
|
1384
|
-
|
1385
|
-
|
1386
|
-
|
1387
|
-
|
1388
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1389
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1390
|
-
"""
|
1391
|
-
...
|
1392
|
-
|
1393
|
-
@typing.overload
|
1394
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1395
|
-
"""
|
1396
|
-
Specifies the times when the flow should be run when running on a
|
1397
|
-
production scheduler.
|
1444
|
+
name : str
|
1445
|
+
Project name. Make sure that the name is unique amongst all
|
1446
|
+
projects that use the same production scheduler. The name may
|
1447
|
+
contain only lowercase alphanumeric characters and underscores.
|
1398
1448
|
|
1399
|
-
Parameters
|
1400
|
-
----------
|
1401
|
-
hourly : bool, default False
|
1402
|
-
Run the workflow hourly.
|
1403
|
-
daily : bool, default True
|
1404
|
-
Run the workflow daily.
|
1405
|
-
weekly : bool, default False
|
1406
|
-
Run the workflow weekly.
|
1407
|
-
cron : str, optional, default None
|
1408
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1409
|
-
specified by this expression.
|
1410
|
-
timezone : str, optional, default None
|
1411
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1412
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1413
|
-
"""
|
1414
|
-
...
|
1415
|
-
|
1416
|
-
@typing.overload
|
1417
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1418
|
-
...
|
1419
|
-
|
1420
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1421
|
-
"""
|
1422
|
-
Specifies the times when the flow should be run when running on a
|
1423
|
-
production scheduler.
|
1424
1449
|
|
1425
|
-
Parameters
|
1426
|
-
----------
|
1427
|
-
hourly : bool, default False
|
1428
|
-
Run the workflow hourly.
|
1429
|
-
daily : bool, default True
|
1430
|
-
Run the workflow daily.
|
1431
|
-
weekly : bool, default False
|
1432
|
-
Run the workflow weekly.
|
1433
|
-
cron : str, optional, default None
|
1434
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1435
|
-
specified by this expression.
|
1436
|
-
timezone : str, optional, default None
|
1437
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1438
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1439
1450
|
"""
|
1440
1451
|
...
|
1441
1452
|
|
@@ -1534,24 +1545,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1534
1545
|
"""
|
1535
1546
|
...
|
1536
1547
|
|
1537
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1538
|
-
"""
|
1539
|
-
Specifies what flows belong to the same project.
|
1540
|
-
|
1541
|
-
A project-specific namespace is created for all flows that
|
1542
|
-
use the same `@project(name)`.
|
1543
|
-
|
1544
|
-
Parameters
|
1545
|
-
----------
|
1546
|
-
name : str
|
1547
|
-
Project name. Make sure that the name is unique amongst all
|
1548
|
-
projects that use the same production scheduler. The name may
|
1549
|
-
contain only lowercase alphanumeric characters and underscores.
|
1550
|
-
|
1551
|
-
|
1552
|
-
"""
|
1553
|
-
...
|
1554
|
-
|
1555
1548
|
@typing.overload
|
1556
1549
|
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1557
1550
|
"""
|
@@ -1601,6 +1594,45 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
|
|
1601
1594
|
"""
|
1602
1595
|
...
|
1603
1596
|
|
1597
|
+
@typing.overload
|
1598
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1599
|
+
"""
|
1600
|
+
Specifies the PyPI packages for all steps of the flow.
|
1601
|
+
|
1602
|
+
Use `@pypi_base` to set common packages required by all
|
1603
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1604
|
+
Parameters
|
1605
|
+
----------
|
1606
|
+
packages : Dict[str, str], default: {}
|
1607
|
+
Packages to use for this flow. The key is the name of the package
|
1608
|
+
and the value is the version to use.
|
1609
|
+
python : str, optional, default: None
|
1610
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1611
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1612
|
+
"""
|
1613
|
+
...
|
1614
|
+
|
1615
|
+
@typing.overload
|
1616
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1617
|
+
...
|
1618
|
+
|
1619
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1620
|
+
"""
|
1621
|
+
Specifies the PyPI packages for all steps of the flow.
|
1622
|
+
|
1623
|
+
Use `@pypi_base` to set common packages required by all
|
1624
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1625
|
+
Parameters
|
1626
|
+
----------
|
1627
|
+
packages : Dict[str, str], default: {}
|
1628
|
+
Packages to use for this flow. The key is the name of the package
|
1629
|
+
and the value is the version to use.
|
1630
|
+
python : str, optional, default: None
|
1631
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1632
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1633
|
+
"""
|
1634
|
+
...
|
1635
|
+
|
1604
1636
|
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1605
1637
|
"""
|
1606
1638
|
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
@@ -1643,84 +1675,52 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1643
1675
|
"""
|
1644
1676
|
...
|
1645
1677
|
|
1646
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1647
|
-
"""
|
1648
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1649
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1650
|
-
|
1651
|
-
Parameters
|
1652
|
-
----------
|
1653
|
-
timeout : int
|
1654
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1655
|
-
poke_interval : int
|
1656
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1657
|
-
mode : str
|
1658
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1659
|
-
exponential_backoff : bool
|
1660
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1661
|
-
pool : str
|
1662
|
-
the slot pool this task should run in,
|
1663
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1664
|
-
soft_fail : bool
|
1665
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1666
|
-
name : str
|
1667
|
-
Name of the sensor on Airflow
|
1668
|
-
description : str
|
1669
|
-
Description of sensor in the Airflow UI
|
1670
|
-
external_dag_id : str
|
1671
|
-
The dag_id that contains the task you want to wait for.
|
1672
|
-
external_task_ids : List[str]
|
1673
|
-
The list of task_ids that you want to wait for.
|
1674
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
1675
|
-
allowed_states : List[str]
|
1676
|
-
Iterable of allowed states, (Default: ['success'])
|
1677
|
-
failed_states : List[str]
|
1678
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
1679
|
-
execution_delta : datetime.timedelta
|
1680
|
-
time difference with the previous execution to look at,
|
1681
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
1682
|
-
check_existence: bool
|
1683
|
-
Set to True to check if the external task exists or check if
|
1684
|
-
the DAG to wait for exists. (Default: True)
|
1685
|
-
"""
|
1686
|
-
...
|
1687
|
-
|
1688
1678
|
@typing.overload
|
1689
|
-
def
|
1679
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1690
1680
|
"""
|
1691
|
-
Specifies the
|
1681
|
+
Specifies the times when the flow should be run when running on a
|
1682
|
+
production scheduler.
|
1692
1683
|
|
1693
|
-
Use `@pypi_base` to set common packages required by all
|
1694
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1695
1684
|
Parameters
|
1696
1685
|
----------
|
1697
|
-
|
1698
|
-
|
1699
|
-
|
1700
|
-
|
1701
|
-
|
1702
|
-
|
1686
|
+
hourly : bool, default False
|
1687
|
+
Run the workflow hourly.
|
1688
|
+
daily : bool, default True
|
1689
|
+
Run the workflow daily.
|
1690
|
+
weekly : bool, default False
|
1691
|
+
Run the workflow weekly.
|
1692
|
+
cron : str, optional, default None
|
1693
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1694
|
+
specified by this expression.
|
1695
|
+
timezone : str, optional, default None
|
1696
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1697
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1703
1698
|
"""
|
1704
1699
|
...
|
1705
1700
|
|
1706
1701
|
@typing.overload
|
1707
|
-
def
|
1702
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1708
1703
|
...
|
1709
1704
|
|
1710
|
-
def
|
1705
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1711
1706
|
"""
|
1712
|
-
Specifies the
|
1707
|
+
Specifies the times when the flow should be run when running on a
|
1708
|
+
production scheduler.
|
1713
1709
|
|
1714
|
-
Use `@pypi_base` to set common packages required by all
|
1715
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1716
1710
|
Parameters
|
1717
1711
|
----------
|
1718
|
-
|
1719
|
-
|
1720
|
-
|
1721
|
-
|
1722
|
-
|
1723
|
-
|
1712
|
+
hourly : bool, default False
|
1713
|
+
Run the workflow hourly.
|
1714
|
+
daily : bool, default True
|
1715
|
+
Run the workflow daily.
|
1716
|
+
weekly : bool, default False
|
1717
|
+
Run the workflow weekly.
|
1718
|
+
cron : str, optional, default None
|
1719
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1720
|
+
specified by this expression.
|
1721
|
+
timezone : str, optional, default None
|
1722
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1723
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1724
1724
|
"""
|
1725
1725
|
...
|
1726
1726
|
|