ob-metaflow-stubs 4.0__py2.py3-none-any.whl → 4.1__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +442 -442
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +4 -4
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +4 -4
- metaflow-stubs/metadata/metadata.pyi +3 -3
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +4 -2
- metaflow-stubs/metaflow_current.pyi +4 -4
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +3 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +7 -3
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +2 -2
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +4 -4
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +4 -4
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-4.0.dist-info → ob_metaflow_stubs-4.1.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-4.1.dist-info/RECORD +142 -0
- ob_metaflow_stubs-4.0.dist-info/RECORD +0 -142
- {ob_metaflow_stubs-4.0.dist-info → ob_metaflow_stubs-4.1.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-4.0.dist-info → ob_metaflow_stubs-4.1.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,25 +1,25 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.6.1+ob(v1) #
|
4
|
+
# Generated on 2024-07-03T18:20:45.077392 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import io
|
12
|
-
import metaflow.client.core
|
13
|
-
import metaflow.parameters
|
14
|
-
import metaflow.metaflow_current
|
15
11
|
import metaflow.flowspec
|
16
|
-
import metaflow.runner.metaflow_runner
|
17
|
-
import typing
|
18
|
-
import datetime
|
19
|
-
import metaflow.plugins.datatools.s3.s3
|
20
12
|
import metaflow.datastore.inputs
|
21
13
|
import metaflow.events
|
22
14
|
import metaflow._vendor.click.types
|
15
|
+
import metaflow.metaflow_current
|
16
|
+
import metaflow.parameters
|
17
|
+
import typing
|
18
|
+
import metaflow.runner.metaflow_runner
|
19
|
+
import io
|
20
|
+
import metaflow.client.core
|
21
|
+
import metaflow.plugins.datatools.s3.s3
|
22
|
+
import datetime
|
23
23
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
24
24
|
StepFlag = typing.NewType("StepFlag", bool)
|
25
25
|
|
@@ -727,6 +727,114 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
727
727
|
"""
|
728
728
|
...
|
729
729
|
|
730
|
+
@typing.overload
|
731
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
732
|
+
"""
|
733
|
+
Specifies that the step will success under all circumstances.
|
734
|
+
|
735
|
+
The decorator will create an optional artifact, specified by `var`, which
|
736
|
+
contains the exception raised. You can use it to detect the presence
|
737
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
738
|
+
are missing.
|
739
|
+
|
740
|
+
Parameters
|
741
|
+
----------
|
742
|
+
var : str, optional, default None
|
743
|
+
Name of the artifact in which to store the caught exception.
|
744
|
+
If not specified, the exception is not stored.
|
745
|
+
print_exception : bool, default True
|
746
|
+
Determines whether or not the exception is printed to
|
747
|
+
stdout when caught.
|
748
|
+
"""
|
749
|
+
...
|
750
|
+
|
751
|
+
@typing.overload
|
752
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
753
|
+
...
|
754
|
+
|
755
|
+
@typing.overload
|
756
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
757
|
+
...
|
758
|
+
|
759
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
760
|
+
"""
|
761
|
+
Specifies that the step will success under all circumstances.
|
762
|
+
|
763
|
+
The decorator will create an optional artifact, specified by `var`, which
|
764
|
+
contains the exception raised. You can use it to detect the presence
|
765
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
766
|
+
are missing.
|
767
|
+
|
768
|
+
Parameters
|
769
|
+
----------
|
770
|
+
var : str, optional, default None
|
771
|
+
Name of the artifact in which to store the caught exception.
|
772
|
+
If not specified, the exception is not stored.
|
773
|
+
print_exception : bool, default True
|
774
|
+
Determines whether or not the exception is printed to
|
775
|
+
stdout when caught.
|
776
|
+
"""
|
777
|
+
...
|
778
|
+
|
779
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
780
|
+
"""
|
781
|
+
Specifies that this step should execute on Kubernetes.
|
782
|
+
|
783
|
+
Parameters
|
784
|
+
----------
|
785
|
+
cpu : int, default 1
|
786
|
+
Number of CPUs required for this step. If `@resources` is
|
787
|
+
also present, the maximum value from all decorators is used.
|
788
|
+
memory : int, default 4096
|
789
|
+
Memory size (in MB) required for this step. If
|
790
|
+
`@resources` is also present, the maximum value from all decorators is
|
791
|
+
used.
|
792
|
+
disk : int, default 10240
|
793
|
+
Disk size (in MB) required for this step. If
|
794
|
+
`@resources` is also present, the maximum value from all decorators is
|
795
|
+
used.
|
796
|
+
image : str, optional, default None
|
797
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
798
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
799
|
+
not, a default Docker image mapping to the current version of Python is used.
|
800
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
801
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
802
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
803
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
804
|
+
secrets : List[str], optional, default None
|
805
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
806
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
807
|
+
in Metaflow configuration.
|
808
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
809
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
810
|
+
gpu : int, optional, default None
|
811
|
+
Number of GPUs required for this step. A value of zero implies that
|
812
|
+
the scheduled node should not have GPUs.
|
813
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
814
|
+
The vendor of the GPUs to be used for this step.
|
815
|
+
tolerations : List[str], default []
|
816
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
817
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
818
|
+
use_tmpfs : bool, default False
|
819
|
+
This enables an explicit tmpfs mount for this step.
|
820
|
+
tmpfs_tempdir : bool, default True
|
821
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
822
|
+
tmpfs_size : int, optional, default: None
|
823
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
824
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
825
|
+
memory allocated for this step.
|
826
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
827
|
+
Path to tmpfs mount for this step.
|
828
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
829
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
830
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
831
|
+
shared_memory: int, optional
|
832
|
+
Shared memory size (in MiB) required for this step
|
833
|
+
port: int, optional
|
834
|
+
Port number to specify in the Kubernetes job object
|
835
|
+
"""
|
836
|
+
...
|
837
|
+
|
730
838
|
@typing.overload
|
731
839
|
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
732
840
|
"""
|
@@ -778,55 +886,6 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
778
886
|
"""
|
779
887
|
...
|
780
888
|
|
781
|
-
@typing.overload
|
782
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
783
|
-
"""
|
784
|
-
Specifies the PyPI packages for the step.
|
785
|
-
|
786
|
-
Information in this decorator will augment any
|
787
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
788
|
-
you can use `@pypi_base` to set packages required by all
|
789
|
-
steps and use `@pypi` to specify step-specific overrides.
|
790
|
-
|
791
|
-
Parameters
|
792
|
-
----------
|
793
|
-
packages : Dict[str, str], default: {}
|
794
|
-
Packages to use for this step. The key is the name of the package
|
795
|
-
and the value is the version to use.
|
796
|
-
python : str, optional, default: None
|
797
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
798
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
799
|
-
"""
|
800
|
-
...
|
801
|
-
|
802
|
-
@typing.overload
|
803
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
804
|
-
...
|
805
|
-
|
806
|
-
@typing.overload
|
807
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
808
|
-
...
|
809
|
-
|
810
|
-
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
811
|
-
"""
|
812
|
-
Specifies the PyPI packages for the step.
|
813
|
-
|
814
|
-
Information in this decorator will augment any
|
815
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
816
|
-
you can use `@pypi_base` to set packages required by all
|
817
|
-
steps and use `@pypi` to specify step-specific overrides.
|
818
|
-
|
819
|
-
Parameters
|
820
|
-
----------
|
821
|
-
packages : Dict[str, str], default: {}
|
822
|
-
Packages to use for this step. The key is the name of the package
|
823
|
-
and the value is the version to use.
|
824
|
-
python : str, optional, default: None
|
825
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
826
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
827
|
-
"""
|
828
|
-
...
|
829
|
-
|
830
889
|
@typing.overload
|
831
890
|
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
832
891
|
"""
|
@@ -884,55 +943,6 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
884
943
|
"""
|
885
944
|
...
|
886
945
|
|
887
|
-
@typing.overload
|
888
|
-
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
889
|
-
"""
|
890
|
-
Specifies that the step will success under all circumstances.
|
891
|
-
|
892
|
-
The decorator will create an optional artifact, specified by `var`, which
|
893
|
-
contains the exception raised. You can use it to detect the presence
|
894
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
895
|
-
are missing.
|
896
|
-
|
897
|
-
Parameters
|
898
|
-
----------
|
899
|
-
var : str, optional, default None
|
900
|
-
Name of the artifact in which to store the caught exception.
|
901
|
-
If not specified, the exception is not stored.
|
902
|
-
print_exception : bool, default True
|
903
|
-
Determines whether or not the exception is printed to
|
904
|
-
stdout when caught.
|
905
|
-
"""
|
906
|
-
...
|
907
|
-
|
908
|
-
@typing.overload
|
909
|
-
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
910
|
-
...
|
911
|
-
|
912
|
-
@typing.overload
|
913
|
-
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
914
|
-
...
|
915
|
-
|
916
|
-
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
917
|
-
"""
|
918
|
-
Specifies that the step will success under all circumstances.
|
919
|
-
|
920
|
-
The decorator will create an optional artifact, specified by `var`, which
|
921
|
-
contains the exception raised. You can use it to detect the presence
|
922
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
923
|
-
are missing.
|
924
|
-
|
925
|
-
Parameters
|
926
|
-
----------
|
927
|
-
var : str, optional, default None
|
928
|
-
Name of the artifact in which to store the caught exception.
|
929
|
-
If not specified, the exception is not stored.
|
930
|
-
print_exception : bool, default True
|
931
|
-
Determines whether or not the exception is printed to
|
932
|
-
stdout when caught.
|
933
|
-
"""
|
934
|
-
...
|
935
|
-
|
936
946
|
@typing.overload
|
937
947
|
def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
938
948
|
"""
|
@@ -1080,37 +1090,6 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1080
1090
|
"""
|
1081
1091
|
...
|
1082
1092
|
|
1083
|
-
@typing.overload
|
1084
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1085
|
-
"""
|
1086
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1087
|
-
|
1088
|
-
Parameters
|
1089
|
-
----------
|
1090
|
-
vars : Dict[str, str], default {}
|
1091
|
-
Dictionary of environment variables to set.
|
1092
|
-
"""
|
1093
|
-
...
|
1094
|
-
|
1095
|
-
@typing.overload
|
1096
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1097
|
-
...
|
1098
|
-
|
1099
|
-
@typing.overload
|
1100
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1101
|
-
...
|
1102
|
-
|
1103
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1104
|
-
"""
|
1105
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1106
|
-
|
1107
|
-
Parameters
|
1108
|
-
----------
|
1109
|
-
vars : Dict[str, str], default {}
|
1110
|
-
Dictionary of environment variables to set.
|
1111
|
-
"""
|
1112
|
-
...
|
1113
|
-
|
1114
1093
|
@typing.overload
|
1115
1094
|
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1116
1095
|
"""
|
@@ -1164,124 +1143,8 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1164
1143
|
"""
|
1165
1144
|
...
|
1166
1145
|
|
1167
|
-
|
1168
|
-
|
1169
|
-
Specifies that this step should execute on Kubernetes.
|
1170
|
-
|
1171
|
-
Parameters
|
1172
|
-
----------
|
1173
|
-
cpu : int, default 1
|
1174
|
-
Number of CPUs required for this step. If `@resources` is
|
1175
|
-
also present, the maximum value from all decorators is used.
|
1176
|
-
memory : int, default 4096
|
1177
|
-
Memory size (in MB) required for this step. If
|
1178
|
-
`@resources` is also present, the maximum value from all decorators is
|
1179
|
-
used.
|
1180
|
-
disk : int, default 10240
|
1181
|
-
Disk size (in MB) required for this step. If
|
1182
|
-
`@resources` is also present, the maximum value from all decorators is
|
1183
|
-
used.
|
1184
|
-
image : str, optional, default None
|
1185
|
-
Docker image to use when launching on Kubernetes. If not specified, and
|
1186
|
-
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
1187
|
-
not, a default Docker image mapping to the current version of Python is used.
|
1188
|
-
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
1189
|
-
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
1190
|
-
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
1191
|
-
Kubernetes service account to use when launching pod in Kubernetes.
|
1192
|
-
secrets : List[str], optional, default None
|
1193
|
-
Kubernetes secrets to use when launching pod in Kubernetes. These
|
1194
|
-
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
1195
|
-
in Metaflow configuration.
|
1196
|
-
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
1197
|
-
Kubernetes namespace to use when launching pod in Kubernetes.
|
1198
|
-
gpu : int, optional, default None
|
1199
|
-
Number of GPUs required for this step. A value of zero implies that
|
1200
|
-
the scheduled node should not have GPUs.
|
1201
|
-
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
1202
|
-
The vendor of the GPUs to be used for this step.
|
1203
|
-
tolerations : List[str], default []
|
1204
|
-
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
1205
|
-
Kubernetes tolerations to use when launching pod in Kubernetes.
|
1206
|
-
use_tmpfs : bool, default False
|
1207
|
-
This enables an explicit tmpfs mount for this step.
|
1208
|
-
tmpfs_tempdir : bool, default True
|
1209
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
1210
|
-
tmpfs_size : int, optional, default: None
|
1211
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
1212
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
1213
|
-
memory allocated for this step.
|
1214
|
-
tmpfs_path : str, optional, default /metaflow_temp
|
1215
|
-
Path to tmpfs mount for this step.
|
1216
|
-
persistent_volume_claims : Dict[str, str], optional, default None
|
1217
|
-
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
1218
|
-
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
1219
|
-
shared_memory: int, optional
|
1220
|
-
Shared memory size (in MiB) required for this step
|
1221
|
-
port: int, optional
|
1222
|
-
Port number to specify in the Kubernetes job object
|
1223
|
-
"""
|
1224
|
-
...
|
1225
|
-
|
1226
|
-
@typing.overload
|
1227
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1228
|
-
"""
|
1229
|
-
Specifies the Conda environment for the step.
|
1230
|
-
|
1231
|
-
Information in this decorator will augment any
|
1232
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1233
|
-
you can use `@conda_base` to set packages required by all
|
1234
|
-
steps and use `@conda` to specify step-specific overrides.
|
1235
|
-
|
1236
|
-
Parameters
|
1237
|
-
----------
|
1238
|
-
packages : Dict[str, str], default {}
|
1239
|
-
Packages to use for this step. The key is the name of the package
|
1240
|
-
and the value is the version to use.
|
1241
|
-
libraries : Dict[str, str], default {}
|
1242
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1243
|
-
python : str, optional, default None
|
1244
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1245
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1246
|
-
disabled : bool, default False
|
1247
|
-
If set to True, disables @conda.
|
1248
|
-
"""
|
1249
|
-
...
|
1250
|
-
|
1251
|
-
@typing.overload
|
1252
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1253
|
-
...
|
1254
|
-
|
1255
|
-
@typing.overload
|
1256
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1257
|
-
...
|
1258
|
-
|
1259
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1260
|
-
"""
|
1261
|
-
Specifies the Conda environment for the step.
|
1262
|
-
|
1263
|
-
Information in this decorator will augment any
|
1264
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1265
|
-
you can use `@conda_base` to set packages required by all
|
1266
|
-
steps and use `@conda` to specify step-specific overrides.
|
1267
|
-
|
1268
|
-
Parameters
|
1269
|
-
----------
|
1270
|
-
packages : Dict[str, str], default {}
|
1271
|
-
Packages to use for this step. The key is the name of the package
|
1272
|
-
and the value is the version to use.
|
1273
|
-
libraries : Dict[str, str], default {}
|
1274
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1275
|
-
python : str, optional, default None
|
1276
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1277
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1278
|
-
disabled : bool, default False
|
1279
|
-
If set to True, disables @conda.
|
1280
|
-
"""
|
1281
|
-
...
|
1282
|
-
|
1283
|
-
@typing.overload
|
1284
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1146
|
+
@typing.overload
|
1147
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1285
1148
|
"""
|
1286
1149
|
Specifies secrets to be retrieved and injected as environment variables prior to
|
1287
1150
|
the execution of a step.
|
@@ -1391,139 +1254,146 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
1391
1254
|
...
|
1392
1255
|
|
1393
1256
|
@typing.overload
|
1394
|
-
def
|
1257
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1395
1258
|
"""
|
1396
|
-
Specifies the
|
1259
|
+
Specifies the Conda environment for the step.
|
1260
|
+
|
1261
|
+
Information in this decorator will augment any
|
1262
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1263
|
+
you can use `@conda_base` to set packages required by all
|
1264
|
+
steps and use `@conda` to specify step-specific overrides.
|
1397
1265
|
|
1398
|
-
Use `@pypi_base` to set common packages required by all
|
1399
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1400
1266
|
Parameters
|
1401
1267
|
----------
|
1402
|
-
packages : Dict[str, str], default
|
1403
|
-
Packages to use for this
|
1268
|
+
packages : Dict[str, str], default {}
|
1269
|
+
Packages to use for this step. The key is the name of the package
|
1404
1270
|
and the value is the version to use.
|
1405
|
-
|
1271
|
+
libraries : Dict[str, str], default {}
|
1272
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1273
|
+
python : str, optional, default None
|
1406
1274
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1407
1275
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1276
|
+
disabled : bool, default False
|
1277
|
+
If set to True, disables @conda.
|
1408
1278
|
"""
|
1409
1279
|
...
|
1410
1280
|
|
1411
1281
|
@typing.overload
|
1412
|
-
def
|
1282
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1413
1283
|
...
|
1414
1284
|
|
1415
|
-
|
1285
|
+
@typing.overload
|
1286
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1287
|
+
...
|
1288
|
+
|
1289
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1416
1290
|
"""
|
1417
|
-
Specifies the
|
1291
|
+
Specifies the Conda environment for the step.
|
1292
|
+
|
1293
|
+
Information in this decorator will augment any
|
1294
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1295
|
+
you can use `@conda_base` to set packages required by all
|
1296
|
+
steps and use `@conda` to specify step-specific overrides.
|
1418
1297
|
|
1419
|
-
Use `@pypi_base` to set common packages required by all
|
1420
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1421
1298
|
Parameters
|
1422
1299
|
----------
|
1423
|
-
packages : Dict[str, str], default
|
1424
|
-
Packages to use for this
|
1300
|
+
packages : Dict[str, str], default {}
|
1301
|
+
Packages to use for this step. The key is the name of the package
|
1425
1302
|
and the value is the version to use.
|
1426
|
-
|
1303
|
+
libraries : Dict[str, str], default {}
|
1304
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1305
|
+
python : str, optional, default None
|
1427
1306
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1428
1307
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1308
|
+
disabled : bool, default False
|
1309
|
+
If set to True, disables @conda.
|
1429
1310
|
"""
|
1430
1311
|
...
|
1431
1312
|
|
1432
1313
|
@typing.overload
|
1433
|
-
def
|
1314
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1434
1315
|
"""
|
1435
|
-
Specifies
|
1316
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1436
1317
|
|
1437
|
-
|
1438
|
-
|
1318
|
+
Parameters
|
1319
|
+
----------
|
1320
|
+
vars : Dict[str, str], default {}
|
1321
|
+
Dictionary of environment variables to set.
|
1322
|
+
"""
|
1323
|
+
...
|
1324
|
+
|
1325
|
+
@typing.overload
|
1326
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1327
|
+
...
|
1328
|
+
|
1329
|
+
@typing.overload
|
1330
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1331
|
+
...
|
1332
|
+
|
1333
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1334
|
+
"""
|
1335
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1439
1336
|
|
1440
1337
|
Parameters
|
1441
1338
|
----------
|
1442
|
-
|
1443
|
-
|
1339
|
+
vars : Dict[str, str], default {}
|
1340
|
+
Dictionary of environment variables to set.
|
1341
|
+
"""
|
1342
|
+
...
|
1343
|
+
|
1344
|
+
@typing.overload
|
1345
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1346
|
+
"""
|
1347
|
+
Specifies the PyPI packages for the step.
|
1348
|
+
|
1349
|
+
Information in this decorator will augment any
|
1350
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1351
|
+
you can use `@pypi_base` to set packages required by all
|
1352
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1353
|
+
|
1354
|
+
Parameters
|
1355
|
+
----------
|
1356
|
+
packages : Dict[str, str], default: {}
|
1357
|
+
Packages to use for this step. The key is the name of the package
|
1444
1358
|
and the value is the version to use.
|
1445
|
-
|
1446
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1447
|
-
python : str, optional, default None
|
1359
|
+
python : str, optional, default: None
|
1448
1360
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1449
1361
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1450
|
-
disabled : bool, default False
|
1451
|
-
If set to True, disables Conda.
|
1452
1362
|
"""
|
1453
1363
|
...
|
1454
1364
|
|
1455
1365
|
@typing.overload
|
1456
|
-
def
|
1366
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1457
1367
|
...
|
1458
1368
|
|
1459
|
-
|
1369
|
+
@typing.overload
|
1370
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1371
|
+
...
|
1372
|
+
|
1373
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1460
1374
|
"""
|
1461
|
-
Specifies the
|
1375
|
+
Specifies the PyPI packages for the step.
|
1462
1376
|
|
1463
|
-
|
1464
|
-
|
1377
|
+
Information in this decorator will augment any
|
1378
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1379
|
+
you can use `@pypi_base` to set packages required by all
|
1380
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1465
1381
|
|
1466
1382
|
Parameters
|
1467
1383
|
----------
|
1468
|
-
packages : Dict[str, str], default {}
|
1469
|
-
Packages to use for this
|
1384
|
+
packages : Dict[str, str], default: {}
|
1385
|
+
Packages to use for this step. The key is the name of the package
|
1470
1386
|
and the value is the version to use.
|
1471
|
-
|
1472
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1473
|
-
python : str, optional, default None
|
1387
|
+
python : str, optional, default: None
|
1474
1388
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1475
1389
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1476
|
-
disabled : bool, default False
|
1477
|
-
If set to True, disables Conda.
|
1478
1390
|
"""
|
1479
1391
|
...
|
1480
1392
|
|
1481
|
-
def
|
1393
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1482
1394
|
"""
|
1483
|
-
The `@
|
1484
|
-
|
1485
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1486
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1487
|
-
starts only after all sensors finish.
|
1488
|
-
|
1489
|
-
Parameters
|
1490
|
-
----------
|
1491
|
-
timeout : int
|
1492
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1493
|
-
poke_interval : int
|
1494
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1495
|
-
mode : str
|
1496
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1497
|
-
exponential_backoff : bool
|
1498
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1499
|
-
pool : str
|
1500
|
-
the slot pool this task should run in,
|
1501
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1502
|
-
soft_fail : bool
|
1503
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1504
|
-
name : str
|
1505
|
-
Name of the sensor on Airflow
|
1506
|
-
description : str
|
1507
|
-
Description of sensor in the Airflow UI
|
1508
|
-
bucket_key : Union[str, List[str]]
|
1509
|
-
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1510
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1511
|
-
bucket_name : str
|
1512
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1513
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1514
|
-
wildcard_match : bool
|
1515
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1516
|
-
aws_conn_id : str
|
1517
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
1518
|
-
verify : bool
|
1519
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1520
|
-
"""
|
1521
|
-
...
|
1522
|
-
|
1523
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1524
|
-
"""
|
1525
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1526
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1395
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1396
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1527
1397
|
|
1528
1398
|
Parameters
|
1529
1399
|
----------
|
@@ -1563,97 +1433,139 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1563
1433
|
...
|
1564
1434
|
|
1565
1435
|
@typing.overload
|
1566
|
-
def
|
1436
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1567
1437
|
"""
|
1568
|
-
Specifies the
|
1569
|
-
|
1570
|
-
```
|
1571
|
-
@trigger(event='foo')
|
1572
|
-
```
|
1573
|
-
or
|
1574
|
-
```
|
1575
|
-
@trigger(events=['foo', 'bar'])
|
1576
|
-
```
|
1577
|
-
|
1578
|
-
Additionally, you can specify the parameter mappings
|
1579
|
-
to map event payload to Metaflow parameters for the flow.
|
1580
|
-
```
|
1581
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1582
|
-
```
|
1583
|
-
or
|
1584
|
-
```
|
1585
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1586
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1587
|
-
```
|
1438
|
+
Specifies the PyPI packages for all steps of the flow.
|
1588
1439
|
|
1589
|
-
|
1590
|
-
|
1591
|
-
|
1592
|
-
|
1593
|
-
|
1594
|
-
|
1595
|
-
|
1596
|
-
|
1440
|
+
Use `@pypi_base` to set common packages required by all
|
1441
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1442
|
+
Parameters
|
1443
|
+
----------
|
1444
|
+
packages : Dict[str, str], default: {}
|
1445
|
+
Packages to use for this flow. The key is the name of the package
|
1446
|
+
and the value is the version to use.
|
1447
|
+
python : str, optional, default: None
|
1448
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1449
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1450
|
+
"""
|
1451
|
+
...
|
1452
|
+
|
1453
|
+
@typing.overload
|
1454
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1455
|
+
...
|
1456
|
+
|
1457
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1458
|
+
"""
|
1459
|
+
Specifies the PyPI packages for all steps of the flow.
|
1597
1460
|
|
1461
|
+
Use `@pypi_base` to set common packages required by all
|
1462
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1598
1463
|
Parameters
|
1599
1464
|
----------
|
1600
|
-
|
1601
|
-
|
1602
|
-
|
1603
|
-
|
1604
|
-
|
1605
|
-
|
1465
|
+
packages : Dict[str, str], default: {}
|
1466
|
+
Packages to use for this flow. The key is the name of the package
|
1467
|
+
and the value is the version to use.
|
1468
|
+
python : str, optional, default: None
|
1469
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1470
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1471
|
+
"""
|
1472
|
+
...
|
1473
|
+
|
1474
|
+
@typing.overload
|
1475
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1476
|
+
"""
|
1477
|
+
Specifies the Conda environment for all steps of the flow.
|
1606
1478
|
|
1479
|
+
Use `@conda_base` to set common libraries required by all
|
1480
|
+
steps and use `@conda` to specify step-specific additions.
|
1607
1481
|
|
1482
|
+
Parameters
|
1483
|
+
----------
|
1484
|
+
packages : Dict[str, str], default {}
|
1485
|
+
Packages to use for this flow. The key is the name of the package
|
1486
|
+
and the value is the version to use.
|
1487
|
+
libraries : Dict[str, str], default {}
|
1488
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1489
|
+
python : str, optional, default None
|
1490
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1491
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1492
|
+
disabled : bool, default False
|
1493
|
+
If set to True, disables Conda.
|
1608
1494
|
"""
|
1609
1495
|
...
|
1610
1496
|
|
1611
1497
|
@typing.overload
|
1612
|
-
def
|
1498
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1613
1499
|
...
|
1614
1500
|
|
1615
|
-
def
|
1501
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1616
1502
|
"""
|
1617
|
-
Specifies the
|
1618
|
-
|
1619
|
-
```
|
1620
|
-
@trigger(event='foo')
|
1621
|
-
```
|
1622
|
-
or
|
1623
|
-
```
|
1624
|
-
@trigger(events=['foo', 'bar'])
|
1625
|
-
```
|
1626
|
-
|
1627
|
-
Additionally, you can specify the parameter mappings
|
1628
|
-
to map event payload to Metaflow parameters for the flow.
|
1629
|
-
```
|
1630
|
-
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1631
|
-
```
|
1632
|
-
or
|
1633
|
-
```
|
1634
|
-
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1635
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1636
|
-
```
|
1503
|
+
Specifies the Conda environment for all steps of the flow.
|
1637
1504
|
|
1638
|
-
|
1639
|
-
|
1640
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1641
|
-
```
|
1642
|
-
This is equivalent to:
|
1643
|
-
```
|
1644
|
-
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1645
|
-
```
|
1505
|
+
Use `@conda_base` to set common libraries required by all
|
1506
|
+
steps and use `@conda` to specify step-specific additions.
|
1646
1507
|
|
1647
1508
|
Parameters
|
1648
1509
|
----------
|
1649
|
-
|
1650
|
-
|
1651
|
-
|
1652
|
-
|
1653
|
-
|
1654
|
-
|
1510
|
+
packages : Dict[str, str], default {}
|
1511
|
+
Packages to use for this flow. The key is the name of the package
|
1512
|
+
and the value is the version to use.
|
1513
|
+
libraries : Dict[str, str], default {}
|
1514
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1515
|
+
python : str, optional, default None
|
1516
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1517
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1518
|
+
disabled : bool, default False
|
1519
|
+
If set to True, disables Conda.
|
1520
|
+
"""
|
1521
|
+
...
|
1522
|
+
|
1523
|
+
@typing.overload
|
1524
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1525
|
+
"""
|
1526
|
+
Specifies the times when the flow should be run when running on a
|
1527
|
+
production scheduler.
|
1655
1528
|
|
1529
|
+
Parameters
|
1530
|
+
----------
|
1531
|
+
hourly : bool, default False
|
1532
|
+
Run the workflow hourly.
|
1533
|
+
daily : bool, default True
|
1534
|
+
Run the workflow daily.
|
1535
|
+
weekly : bool, default False
|
1536
|
+
Run the workflow weekly.
|
1537
|
+
cron : str, optional, default None
|
1538
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1539
|
+
specified by this expression.
|
1540
|
+
timezone : str, optional, default None
|
1541
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1542
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1543
|
+
"""
|
1544
|
+
...
|
1545
|
+
|
1546
|
+
@typing.overload
|
1547
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1548
|
+
...
|
1549
|
+
|
1550
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1551
|
+
"""
|
1552
|
+
Specifies the times when the flow should be run when running on a
|
1553
|
+
production scheduler.
|
1656
1554
|
|
1555
|
+
Parameters
|
1556
|
+
----------
|
1557
|
+
hourly : bool, default False
|
1558
|
+
Run the workflow hourly.
|
1559
|
+
daily : bool, default True
|
1560
|
+
Run the workflow daily.
|
1561
|
+
weekly : bool, default False
|
1562
|
+
Run the workflow weekly.
|
1563
|
+
cron : str, optional, default None
|
1564
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1565
|
+
specified by this expression.
|
1566
|
+
timezone : str, optional, default None
|
1567
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1568
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1657
1569
|
"""
|
1658
1570
|
...
|
1659
1571
|
|
@@ -1761,51 +1673,97 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1761
1673
|
...
|
1762
1674
|
|
1763
1675
|
@typing.overload
|
1764
|
-
def
|
1676
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1765
1677
|
"""
|
1766
|
-
Specifies the
|
1767
|
-
|
1678
|
+
Specifies the event(s) that this flow depends on.
|
1679
|
+
|
1680
|
+
```
|
1681
|
+
@trigger(event='foo')
|
1682
|
+
```
|
1683
|
+
or
|
1684
|
+
```
|
1685
|
+
@trigger(events=['foo', 'bar'])
|
1686
|
+
```
|
1687
|
+
|
1688
|
+
Additionally, you can specify the parameter mappings
|
1689
|
+
to map event payload to Metaflow parameters for the flow.
|
1690
|
+
```
|
1691
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1692
|
+
```
|
1693
|
+
or
|
1694
|
+
```
|
1695
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1696
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1697
|
+
```
|
1698
|
+
|
1699
|
+
'parameters' can also be a list of strings and tuples like so:
|
1700
|
+
```
|
1701
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1702
|
+
```
|
1703
|
+
This is equivalent to:
|
1704
|
+
```
|
1705
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1706
|
+
```
|
1768
1707
|
|
1769
1708
|
Parameters
|
1770
1709
|
----------
|
1771
|
-
|
1772
|
-
|
1773
|
-
|
1774
|
-
|
1775
|
-
|
1776
|
-
|
1777
|
-
|
1778
|
-
|
1779
|
-
specified by this expression.
|
1780
|
-
timezone : str, optional, default None
|
1781
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1782
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1710
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1711
|
+
Event dependency for this flow.
|
1712
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1713
|
+
Events dependency for this flow.
|
1714
|
+
options : Dict[str, Any], default {}
|
1715
|
+
Backend-specific configuration for tuning eventing behavior.
|
1716
|
+
|
1717
|
+
|
1783
1718
|
"""
|
1784
1719
|
...
|
1785
1720
|
|
1786
1721
|
@typing.overload
|
1787
|
-
def
|
1722
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1788
1723
|
...
|
1789
1724
|
|
1790
|
-
def
|
1725
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1791
1726
|
"""
|
1792
|
-
Specifies the
|
1793
|
-
|
1727
|
+
Specifies the event(s) that this flow depends on.
|
1728
|
+
|
1729
|
+
```
|
1730
|
+
@trigger(event='foo')
|
1731
|
+
```
|
1732
|
+
or
|
1733
|
+
```
|
1734
|
+
@trigger(events=['foo', 'bar'])
|
1735
|
+
```
|
1736
|
+
|
1737
|
+
Additionally, you can specify the parameter mappings
|
1738
|
+
to map event payload to Metaflow parameters for the flow.
|
1739
|
+
```
|
1740
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1741
|
+
```
|
1742
|
+
or
|
1743
|
+
```
|
1744
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1745
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1746
|
+
```
|
1747
|
+
|
1748
|
+
'parameters' can also be a list of strings and tuples like so:
|
1749
|
+
```
|
1750
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1751
|
+
```
|
1752
|
+
This is equivalent to:
|
1753
|
+
```
|
1754
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1755
|
+
```
|
1794
1756
|
|
1795
1757
|
Parameters
|
1796
1758
|
----------
|
1797
|
-
|
1798
|
-
|
1799
|
-
|
1800
|
-
|
1801
|
-
|
1802
|
-
|
1803
|
-
|
1804
|
-
|
1805
|
-
specified by this expression.
|
1806
|
-
timezone : str, optional, default None
|
1807
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1808
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1759
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1760
|
+
Event dependency for this flow.
|
1761
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1762
|
+
Events dependency for this flow.
|
1763
|
+
options : Dict[str, Any], default {}
|
1764
|
+
Backend-specific configuration for tuning eventing behavior.
|
1765
|
+
|
1766
|
+
|
1809
1767
|
"""
|
1810
1768
|
...
|
1811
1769
|
|
@@ -1827,6 +1785,48 @@ def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typ
|
|
1827
1785
|
"""
|
1828
1786
|
...
|
1829
1787
|
|
1788
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1789
|
+
"""
|
1790
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1791
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1792
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1793
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1794
|
+
starts only after all sensors finish.
|
1795
|
+
|
1796
|
+
Parameters
|
1797
|
+
----------
|
1798
|
+
timeout : int
|
1799
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1800
|
+
poke_interval : int
|
1801
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1802
|
+
mode : str
|
1803
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1804
|
+
exponential_backoff : bool
|
1805
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1806
|
+
pool : str
|
1807
|
+
the slot pool this task should run in,
|
1808
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1809
|
+
soft_fail : bool
|
1810
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1811
|
+
name : str
|
1812
|
+
Name of the sensor on Airflow
|
1813
|
+
description : str
|
1814
|
+
Description of sensor in the Airflow UI
|
1815
|
+
bucket_key : Union[str, List[str]]
|
1816
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1817
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1818
|
+
bucket_name : str
|
1819
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1820
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1821
|
+
wildcard_match : bool
|
1822
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1823
|
+
aws_conn_id : str
|
1824
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1825
|
+
verify : bool
|
1826
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1827
|
+
"""
|
1828
|
+
...
|
1829
|
+
|
1830
1830
|
def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
1831
1831
|
"""
|
1832
1832
|
Switch namespace to the one provided.
|