metaflow-stubs 2.12.21__py2.py3-none-any.whl → 2.12.22__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +422 -422
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +3 -3
- metaflow-stubs/client/core.pyi +7 -7
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +6 -6
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +4 -4
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata/metadata.pyi +2 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +22 -22
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +3 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +4 -4
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +7 -7
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +4 -4
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +2 -2
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +5 -5
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +4 -4
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +4 -4
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +3 -3
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.12.21.dist-info → metaflow_stubs-2.12.22.dist-info}/METADATA +2 -2
- metaflow_stubs-2.12.22.dist-info/RECORD +152 -0
- metaflow_stubs-2.12.21.dist-info/RECORD +0 -152
- {metaflow_stubs-2.12.21.dist-info → metaflow_stubs-2.12.22.dist-info}/WHEEL +0 -0
- {metaflow_stubs-2.12.21.dist-info → metaflow_stubs-2.12.22.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,25 +1,25 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-09-
|
3
|
+
# MF version: 2.12.22 #
|
4
|
+
# Generated on 2024-09-20T00:45:49.586219 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.
|
12
|
-
import datetime
|
13
|
-
import metaflow.plugins.datatools.s3.s3
|
11
|
+
import metaflow.flowspec
|
14
12
|
import metaflow.runner.metaflow_runner
|
15
|
-
import metaflow.client.core
|
16
13
|
import metaflow.events
|
14
|
+
import metaflow.client.core
|
15
|
+
import metaflow.plugins.datatools.s3.s3
|
17
16
|
import metaflow.metaflow_current
|
18
|
-
import
|
19
|
-
import metaflow.
|
20
|
-
import metaflow._vendor.click.types
|
17
|
+
import datetime
|
18
|
+
import metaflow.datastore.inputs
|
21
19
|
import typing
|
22
20
|
import io
|
21
|
+
import metaflow._vendor.click.types
|
22
|
+
import metaflow.parameters
|
23
23
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
24
24
|
StepFlag = typing.NewType("StepFlag", bool)
|
25
25
|
|
@@ -724,123 +724,79 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
724
724
|
...
|
725
725
|
|
726
726
|
@typing.overload
|
727
|
-
def
|
728
|
-
"""
|
729
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
730
|
-
the execution of a step.
|
731
|
-
|
732
|
-
Parameters
|
733
|
-
----------
|
734
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
735
|
-
List of secret specs, defining how the secrets are to be retrieved
|
736
|
-
"""
|
737
|
-
...
|
738
|
-
|
739
|
-
@typing.overload
|
740
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
741
|
-
...
|
742
|
-
|
743
|
-
@typing.overload
|
744
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
745
|
-
...
|
746
|
-
|
747
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
748
|
-
"""
|
749
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
750
|
-
the execution of a step.
|
751
|
-
|
752
|
-
Parameters
|
753
|
-
----------
|
754
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
755
|
-
List of secret specs, defining how the secrets are to be retrieved
|
756
|
-
"""
|
757
|
-
...
|
758
|
-
|
759
|
-
@typing.overload
|
760
|
-
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
727
|
+
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
761
728
|
"""
|
762
|
-
Specifies
|
763
|
-
|
764
|
-
This decorator is useful if this step may hang indefinitely.
|
729
|
+
Specifies the resources needed when executing this step.
|
765
730
|
|
766
|
-
|
767
|
-
|
768
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
731
|
+
Use `@resources` to specify the resource requirements
|
732
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
769
733
|
|
770
|
-
|
771
|
-
|
734
|
+
You can choose the compute layer on the command line by executing e.g.
|
735
|
+
```
|
736
|
+
python myflow.py run --with batch
|
737
|
+
```
|
738
|
+
or
|
739
|
+
```
|
740
|
+
python myflow.py run --with kubernetes
|
741
|
+
```
|
742
|
+
which executes the flow on the desired system using the
|
743
|
+
requirements specified in `@resources`.
|
772
744
|
|
773
745
|
Parameters
|
774
746
|
----------
|
775
|
-
|
776
|
-
Number of
|
777
|
-
|
778
|
-
Number of
|
779
|
-
|
780
|
-
|
747
|
+
cpu : int, default 1
|
748
|
+
Number of CPUs required for this step.
|
749
|
+
gpu : int, default 0
|
750
|
+
Number of GPUs required for this step.
|
751
|
+
disk : int, optional, default None
|
752
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
753
|
+
memory : int, default 4096
|
754
|
+
Memory size (in MB) required for this step.
|
755
|
+
shared_memory : int, optional, default None
|
756
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
757
|
+
This parameter maps to the `--shm-size` option in Docker.
|
781
758
|
"""
|
782
759
|
...
|
783
760
|
|
784
761
|
@typing.overload
|
785
|
-
def
|
762
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
786
763
|
...
|
787
764
|
|
788
765
|
@typing.overload
|
789
|
-
def
|
766
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
790
767
|
...
|
791
768
|
|
792
|
-
def
|
769
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
793
770
|
"""
|
794
|
-
Specifies
|
795
|
-
|
796
|
-
This decorator is useful if this step may hang indefinitely.
|
797
|
-
|
798
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
799
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
800
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
801
|
-
|
802
|
-
Note that all the values specified in parameters are added together so if you specify
|
803
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
771
|
+
Specifies the resources needed when executing this step.
|
804
772
|
|
805
|
-
|
806
|
-
|
807
|
-
seconds : int, default 0
|
808
|
-
Number of seconds to wait prior to timing out.
|
809
|
-
minutes : int, default 0
|
810
|
-
Number of minutes to wait prior to timing out.
|
811
|
-
hours : int, default 0
|
812
|
-
Number of hours to wait prior to timing out.
|
813
|
-
"""
|
814
|
-
...
|
815
|
-
|
816
|
-
@typing.overload
|
817
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
818
|
-
"""
|
819
|
-
Specifies environment variables to be set prior to the execution of a step.
|
773
|
+
Use `@resources` to specify the resource requirements
|
774
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
820
775
|
|
821
|
-
|
822
|
-
|
823
|
-
|
824
|
-
|
825
|
-
|
826
|
-
|
827
|
-
|
828
|
-
|
829
|
-
|
830
|
-
|
831
|
-
|
832
|
-
@typing.overload
|
833
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
834
|
-
...
|
835
|
-
|
836
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
837
|
-
"""
|
838
|
-
Specifies environment variables to be set prior to the execution of a step.
|
776
|
+
You can choose the compute layer on the command line by executing e.g.
|
777
|
+
```
|
778
|
+
python myflow.py run --with batch
|
779
|
+
```
|
780
|
+
or
|
781
|
+
```
|
782
|
+
python myflow.py run --with kubernetes
|
783
|
+
```
|
784
|
+
which executes the flow on the desired system using the
|
785
|
+
requirements specified in `@resources`.
|
839
786
|
|
840
787
|
Parameters
|
841
788
|
----------
|
842
|
-
|
843
|
-
|
789
|
+
cpu : int, default 1
|
790
|
+
Number of CPUs required for this step.
|
791
|
+
gpu : int, default 0
|
792
|
+
Number of GPUs required for this step.
|
793
|
+
disk : int, optional, default None
|
794
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
795
|
+
memory : int, default 4096
|
796
|
+
Memory size (in MB) required for this step.
|
797
|
+
shared_memory : int, optional, default None
|
798
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
799
|
+
This parameter maps to the `--shm-size` option in Docker.
|
844
800
|
"""
|
845
801
|
...
|
846
802
|
|
@@ -947,99 +903,142 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
947
903
|
...
|
948
904
|
|
949
905
|
@typing.overload
|
950
|
-
def
|
906
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
951
907
|
"""
|
952
|
-
Specifies
|
953
|
-
|
954
|
-
Use `@resources` to specify the resource requirements
|
955
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
956
|
-
|
957
|
-
You can choose the compute layer on the command line by executing e.g.
|
958
|
-
```
|
959
|
-
python myflow.py run --with batch
|
960
|
-
```
|
961
|
-
or
|
962
|
-
```
|
963
|
-
python myflow.py run --with kubernetes
|
964
|
-
```
|
965
|
-
which executes the flow on the desired system using the
|
966
|
-
requirements specified in `@resources`.
|
908
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
909
|
+
the execution of a step.
|
967
910
|
|
968
911
|
Parameters
|
969
912
|
----------
|
970
|
-
|
971
|
-
|
972
|
-
gpu : int, default 0
|
973
|
-
Number of GPUs required for this step.
|
974
|
-
disk : int, optional, default None
|
975
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
976
|
-
memory : int, default 4096
|
977
|
-
Memory size (in MB) required for this step.
|
978
|
-
shared_memory : int, optional, default None
|
979
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
980
|
-
This parameter maps to the `--shm-size` option in Docker.
|
913
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
914
|
+
List of secret specs, defining how the secrets are to be retrieved
|
981
915
|
"""
|
982
916
|
...
|
983
917
|
|
984
918
|
@typing.overload
|
985
|
-
def
|
919
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
986
920
|
...
|
987
921
|
|
988
922
|
@typing.overload
|
989
|
-
def
|
923
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
990
924
|
...
|
991
925
|
|
992
|
-
def
|
926
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
993
927
|
"""
|
994
|
-
Specifies
|
995
|
-
|
996
|
-
Use `@resources` to specify the resource requirements
|
997
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
998
|
-
|
999
|
-
You can choose the compute layer on the command line by executing e.g.
|
1000
|
-
```
|
1001
|
-
python myflow.py run --with batch
|
1002
|
-
```
|
1003
|
-
or
|
1004
|
-
```
|
1005
|
-
python myflow.py run --with kubernetes
|
1006
|
-
```
|
1007
|
-
which executes the flow on the desired system using the
|
1008
|
-
requirements specified in `@resources`.
|
928
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
929
|
+
the execution of a step.
|
1009
930
|
|
1010
931
|
Parameters
|
1011
932
|
----------
|
1012
|
-
|
1013
|
-
|
1014
|
-
gpu : int, default 0
|
1015
|
-
Number of GPUs required for this step.
|
1016
|
-
disk : int, optional, default None
|
1017
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1018
|
-
memory : int, default 4096
|
1019
|
-
Memory size (in MB) required for this step.
|
1020
|
-
shared_memory : int, optional, default None
|
1021
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1022
|
-
This parameter maps to the `--shm-size` option in Docker.
|
933
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
934
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1023
935
|
"""
|
1024
936
|
...
|
1025
937
|
|
1026
938
|
@typing.overload
|
1027
|
-
def
|
939
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1028
940
|
"""
|
1029
|
-
|
1030
|
-
|
941
|
+
Specifies the PyPI packages for the step.
|
942
|
+
|
943
|
+
Information in this decorator will augment any
|
944
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
945
|
+
you can use `@pypi_base` to set packages required by all
|
946
|
+
steps and use `@pypi` to specify step-specific overrides.
|
947
|
+
|
948
|
+
Parameters
|
949
|
+
----------
|
950
|
+
packages : Dict[str, str], default: {}
|
951
|
+
Packages to use for this step. The key is the name of the package
|
952
|
+
and the value is the version to use.
|
953
|
+
python : str, optional, default: None
|
954
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
955
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1031
956
|
"""
|
1032
957
|
...
|
1033
958
|
|
1034
959
|
@typing.overload
|
1035
|
-
def
|
960
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1036
961
|
...
|
1037
962
|
|
1038
|
-
|
1039
|
-
|
1040
|
-
|
1041
|
-
|
1042
|
-
|
963
|
+
@typing.overload
|
964
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
965
|
+
...
|
966
|
+
|
967
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
968
|
+
"""
|
969
|
+
Specifies the PyPI packages for the step.
|
970
|
+
|
971
|
+
Information in this decorator will augment any
|
972
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
973
|
+
you can use `@pypi_base` to set packages required by all
|
974
|
+
steps and use `@pypi` to specify step-specific overrides.
|
975
|
+
|
976
|
+
Parameters
|
977
|
+
----------
|
978
|
+
packages : Dict[str, str], default: {}
|
979
|
+
Packages to use for this step. The key is the name of the package
|
980
|
+
and the value is the version to use.
|
981
|
+
python : str, optional, default: None
|
982
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
983
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
984
|
+
"""
|
985
|
+
...
|
986
|
+
|
987
|
+
@typing.overload
|
988
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
989
|
+
"""
|
990
|
+
Specifies the Conda environment for the step.
|
991
|
+
|
992
|
+
Information in this decorator will augment any
|
993
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
994
|
+
you can use `@conda_base` to set packages required by all
|
995
|
+
steps and use `@conda` to specify step-specific overrides.
|
996
|
+
|
997
|
+
Parameters
|
998
|
+
----------
|
999
|
+
packages : Dict[str, str], default {}
|
1000
|
+
Packages to use for this step. The key is the name of the package
|
1001
|
+
and the value is the version to use.
|
1002
|
+
libraries : Dict[str, str], default {}
|
1003
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1004
|
+
python : str, optional, default None
|
1005
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1006
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1007
|
+
disabled : bool, default False
|
1008
|
+
If set to True, disables @conda.
|
1009
|
+
"""
|
1010
|
+
...
|
1011
|
+
|
1012
|
+
@typing.overload
|
1013
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1014
|
+
...
|
1015
|
+
|
1016
|
+
@typing.overload
|
1017
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1018
|
+
...
|
1019
|
+
|
1020
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1021
|
+
"""
|
1022
|
+
Specifies the Conda environment for the step.
|
1023
|
+
|
1024
|
+
Information in this decorator will augment any
|
1025
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1026
|
+
you can use `@conda_base` to set packages required by all
|
1027
|
+
steps and use `@conda` to specify step-specific overrides.
|
1028
|
+
|
1029
|
+
Parameters
|
1030
|
+
----------
|
1031
|
+
packages : Dict[str, str], default {}
|
1032
|
+
Packages to use for this step. The key is the name of the package
|
1033
|
+
and the value is the version to use.
|
1034
|
+
libraries : Dict[str, str], default {}
|
1035
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1036
|
+
python : str, optional, default None
|
1037
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1038
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1039
|
+
disabled : bool, default False
|
1040
|
+
If set to True, disables @conda.
|
1041
|
+
"""
|
1043
1042
|
...
|
1044
1043
|
|
1045
1044
|
@typing.overload
|
@@ -1190,159 +1189,141 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1190
1189
|
...
|
1191
1190
|
|
1192
1191
|
@typing.overload
|
1193
|
-
def
|
1192
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1194
1193
|
"""
|
1195
|
-
|
1196
|
-
|
1197
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1194
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1198
1195
|
|
1199
1196
|
Parameters
|
1200
1197
|
----------
|
1201
|
-
|
1202
|
-
|
1203
|
-
id : str, optional, default None
|
1204
|
-
If multiple cards are present, use this id to identify this card.
|
1205
|
-
options : Dict[str, Any], default {}
|
1206
|
-
Options passed to the card. The contents depend on the card type.
|
1207
|
-
timeout : int, default 45
|
1208
|
-
Interrupt reporting if it takes more than this many seconds.
|
1209
|
-
|
1210
|
-
|
1198
|
+
vars : Dict[str, str], default {}
|
1199
|
+
Dictionary of environment variables to set.
|
1211
1200
|
"""
|
1212
1201
|
...
|
1213
1202
|
|
1214
1203
|
@typing.overload
|
1215
|
-
def
|
1204
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1216
1205
|
...
|
1217
1206
|
|
1218
1207
|
@typing.overload
|
1219
|
-
def
|
1208
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1220
1209
|
...
|
1221
1210
|
|
1222
|
-
def
|
1211
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1223
1212
|
"""
|
1224
|
-
|
1225
|
-
|
1226
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1213
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1227
1214
|
|
1228
1215
|
Parameters
|
1229
1216
|
----------
|
1230
|
-
|
1231
|
-
|
1232
|
-
id : str, optional, default None
|
1233
|
-
If multiple cards are present, use this id to identify this card.
|
1234
|
-
options : Dict[str, Any], default {}
|
1235
|
-
Options passed to the card. The contents depend on the card type.
|
1236
|
-
timeout : int, default 45
|
1237
|
-
Interrupt reporting if it takes more than this many seconds.
|
1238
|
-
|
1239
|
-
|
1217
|
+
vars : Dict[str, str], default {}
|
1218
|
+
Dictionary of environment variables to set.
|
1240
1219
|
"""
|
1241
1220
|
...
|
1242
1221
|
|
1243
1222
|
@typing.overload
|
1244
|
-
def
|
1223
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1245
1224
|
"""
|
1246
|
-
Specifies
|
1225
|
+
Specifies a timeout for your step.
|
1247
1226
|
|
1248
|
-
|
1249
|
-
|
1250
|
-
|
1251
|
-
|
1227
|
+
This decorator is useful if this step may hang indefinitely.
|
1228
|
+
|
1229
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1230
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1231
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1232
|
+
|
1233
|
+
Note that all the values specified in parameters are added together so if you specify
|
1234
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1252
1235
|
|
1253
1236
|
Parameters
|
1254
1237
|
----------
|
1255
|
-
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1259
|
-
|
1260
|
-
|
1238
|
+
seconds : int, default 0
|
1239
|
+
Number of seconds to wait prior to timing out.
|
1240
|
+
minutes : int, default 0
|
1241
|
+
Number of minutes to wait prior to timing out.
|
1242
|
+
hours : int, default 0
|
1243
|
+
Number of hours to wait prior to timing out.
|
1261
1244
|
"""
|
1262
1245
|
...
|
1263
1246
|
|
1264
1247
|
@typing.overload
|
1265
|
-
def
|
1248
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1266
1249
|
...
|
1267
1250
|
|
1268
1251
|
@typing.overload
|
1269
|
-
def
|
1252
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1270
1253
|
...
|
1271
1254
|
|
1272
|
-
def
|
1255
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1273
1256
|
"""
|
1274
|
-
Specifies
|
1257
|
+
Specifies a timeout for your step.
|
1275
1258
|
|
1276
|
-
|
1277
|
-
|
1278
|
-
|
1279
|
-
|
1259
|
+
This decorator is useful if this step may hang indefinitely.
|
1260
|
+
|
1261
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1262
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1263
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1264
|
+
|
1265
|
+
Note that all the values specified in parameters are added together so if you specify
|
1266
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1280
1267
|
|
1281
1268
|
Parameters
|
1282
1269
|
----------
|
1283
|
-
|
1284
|
-
|
1285
|
-
|
1286
|
-
|
1287
|
-
|
1288
|
-
|
1270
|
+
seconds : int, default 0
|
1271
|
+
Number of seconds to wait prior to timing out.
|
1272
|
+
minutes : int, default 0
|
1273
|
+
Number of minutes to wait prior to timing out.
|
1274
|
+
hours : int, default 0
|
1275
|
+
Number of hours to wait prior to timing out.
|
1289
1276
|
"""
|
1290
1277
|
...
|
1291
1278
|
|
1292
1279
|
@typing.overload
|
1293
|
-
def
|
1280
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1294
1281
|
"""
|
1295
|
-
|
1282
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1296
1283
|
|
1297
|
-
|
1298
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1299
|
-
you can use `@conda_base` to set packages required by all
|
1300
|
-
steps and use `@conda` to specify step-specific overrides.
|
1284
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1301
1285
|
|
1302
1286
|
Parameters
|
1303
1287
|
----------
|
1304
|
-
|
1305
|
-
|
1306
|
-
|
1307
|
-
|
1308
|
-
|
1309
|
-
|
1310
|
-
|
1311
|
-
|
1312
|
-
|
1313
|
-
|
1288
|
+
type : str, default 'default'
|
1289
|
+
Card type.
|
1290
|
+
id : str, optional, default None
|
1291
|
+
If multiple cards are present, use this id to identify this card.
|
1292
|
+
options : Dict[str, Any], default {}
|
1293
|
+
Options passed to the card. The contents depend on the card type.
|
1294
|
+
timeout : int, default 45
|
1295
|
+
Interrupt reporting if it takes more than this many seconds.
|
1296
|
+
|
1297
|
+
|
1314
1298
|
"""
|
1315
1299
|
...
|
1316
1300
|
|
1317
1301
|
@typing.overload
|
1318
|
-
def
|
1302
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1319
1303
|
...
|
1320
1304
|
|
1321
1305
|
@typing.overload
|
1322
|
-
def
|
1306
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1323
1307
|
...
|
1324
1308
|
|
1325
|
-
def
|
1309
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
1326
1310
|
"""
|
1327
|
-
|
1311
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1328
1312
|
|
1329
|
-
|
1330
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1331
|
-
you can use `@conda_base` to set packages required by all
|
1332
|
-
steps and use `@conda` to specify step-specific overrides.
|
1313
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1333
1314
|
|
1334
1315
|
Parameters
|
1335
1316
|
----------
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
|
1341
|
-
|
1342
|
-
|
1343
|
-
|
1344
|
-
|
1345
|
-
|
1317
|
+
type : str, default 'default'
|
1318
|
+
Card type.
|
1319
|
+
id : str, optional, default None
|
1320
|
+
If multiple cards are present, use this id to identify this card.
|
1321
|
+
options : Dict[str, Any], default {}
|
1322
|
+
Options passed to the card. The contents depend on the card type.
|
1323
|
+
timeout : int, default 45
|
1324
|
+
Interrupt reporting if it takes more than this many seconds.
|
1325
|
+
|
1326
|
+
|
1346
1327
|
"""
|
1347
1328
|
...
|
1348
1329
|
|
@@ -1412,45 +1393,22 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
1412
1393
|
"""
|
1413
1394
|
...
|
1414
1395
|
|
1415
|
-
|
1396
|
+
@typing.overload
|
1397
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1416
1398
|
"""
|
1417
|
-
|
1418
|
-
|
1419
|
-
|
1420
|
-
|
1421
|
-
|
1422
|
-
|
1423
|
-
|
1424
|
-
|
1425
|
-
|
1426
|
-
|
1427
|
-
|
1428
|
-
|
1429
|
-
|
1430
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1431
|
-
exponential_backoff : bool
|
1432
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1433
|
-
pool : str
|
1434
|
-
the slot pool this task should run in,
|
1435
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1436
|
-
soft_fail : bool
|
1437
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1438
|
-
name : str
|
1439
|
-
Name of the sensor on Airflow
|
1440
|
-
description : str
|
1441
|
-
Description of sensor in the Airflow UI
|
1442
|
-
bucket_key : Union[str, List[str]]
|
1443
|
-
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1444
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1445
|
-
bucket_name : str
|
1446
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1447
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1448
|
-
wildcard_match : bool
|
1449
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1450
|
-
aws_conn_id : str
|
1451
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
1452
|
-
verify : bool
|
1453
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1399
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1400
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1401
|
+
"""
|
1402
|
+
...
|
1403
|
+
|
1404
|
+
@typing.overload
|
1405
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1406
|
+
...
|
1407
|
+
|
1408
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1409
|
+
"""
|
1410
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1411
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1454
1412
|
"""
|
1455
1413
|
...
|
1456
1414
|
|
@@ -1557,21 +1515,84 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1557
1515
|
"""
|
1558
1516
|
...
|
1559
1517
|
|
1560
|
-
|
1518
|
+
@typing.overload
|
1519
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1561
1520
|
"""
|
1562
|
-
Specifies
|
1563
|
-
|
1564
|
-
A project-specific namespace is created for all flows that
|
1565
|
-
use the same `@project(name)`.
|
1521
|
+
Specifies the PyPI packages for all steps of the flow.
|
1566
1522
|
|
1523
|
+
Use `@pypi_base` to set common packages required by all
|
1524
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1567
1525
|
Parameters
|
1568
1526
|
----------
|
1569
|
-
|
1570
|
-
|
1571
|
-
|
1572
|
-
|
1527
|
+
packages : Dict[str, str], default: {}
|
1528
|
+
Packages to use for this flow. The key is the name of the package
|
1529
|
+
and the value is the version to use.
|
1530
|
+
python : str, optional, default: None
|
1531
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1532
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1533
|
+
"""
|
1534
|
+
...
|
1535
|
+
|
1536
|
+
@typing.overload
|
1537
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1538
|
+
...
|
1539
|
+
|
1540
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1541
|
+
"""
|
1542
|
+
Specifies the PyPI packages for all steps of the flow.
|
1573
1543
|
|
1544
|
+
Use `@pypi_base` to set common packages required by all
|
1545
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1546
|
+
Parameters
|
1547
|
+
----------
|
1548
|
+
packages : Dict[str, str], default: {}
|
1549
|
+
Packages to use for this flow. The key is the name of the package
|
1550
|
+
and the value is the version to use.
|
1551
|
+
python : str, optional, default: None
|
1552
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1553
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1554
|
+
"""
|
1555
|
+
...
|
1556
|
+
|
1557
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1558
|
+
"""
|
1559
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1560
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1574
1561
|
|
1562
|
+
Parameters
|
1563
|
+
----------
|
1564
|
+
timeout : int
|
1565
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1566
|
+
poke_interval : int
|
1567
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1568
|
+
mode : str
|
1569
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1570
|
+
exponential_backoff : bool
|
1571
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1572
|
+
pool : str
|
1573
|
+
the slot pool this task should run in,
|
1574
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1575
|
+
soft_fail : bool
|
1576
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1577
|
+
name : str
|
1578
|
+
Name of the sensor on Airflow
|
1579
|
+
description : str
|
1580
|
+
Description of sensor in the Airflow UI
|
1581
|
+
external_dag_id : str
|
1582
|
+
The dag_id that contains the task you want to wait for.
|
1583
|
+
external_task_ids : List[str]
|
1584
|
+
The list of task_ids that you want to wait for.
|
1585
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1586
|
+
allowed_states : List[str]
|
1587
|
+
Iterable of allowed states, (Default: ['success'])
|
1588
|
+
failed_states : List[str]
|
1589
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1590
|
+
execution_delta : datetime.timedelta
|
1591
|
+
time difference with the previous execution to look at,
|
1592
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1593
|
+
check_existence: bool
|
1594
|
+
Set to True to check if the external task exists or check if
|
1595
|
+
the DAG to wait for exists. (Default: True)
|
1575
1596
|
"""
|
1576
1597
|
...
|
1577
1598
|
|
@@ -1624,91 +1645,45 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
|
|
1624
1645
|
"""
|
1625
1646
|
...
|
1626
1647
|
|
1627
|
-
|
1628
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1629
|
-
"""
|
1630
|
-
Specifies the PyPI packages for all steps of the flow.
|
1631
|
-
|
1632
|
-
Use `@pypi_base` to set common packages required by all
|
1633
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1634
|
-
Parameters
|
1635
|
-
----------
|
1636
|
-
packages : Dict[str, str], default: {}
|
1637
|
-
Packages to use for this flow. The key is the name of the package
|
1638
|
-
and the value is the version to use.
|
1639
|
-
python : str, optional, default: None
|
1640
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1641
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1642
|
-
"""
|
1643
|
-
...
|
1644
|
-
|
1645
|
-
@typing.overload
|
1646
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1647
|
-
...
|
1648
|
-
|
1649
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1650
|
-
"""
|
1651
|
-
Specifies the PyPI packages for all steps of the flow.
|
1652
|
-
|
1653
|
-
Use `@pypi_base` to set common packages required by all
|
1654
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1655
|
-
Parameters
|
1656
|
-
----------
|
1657
|
-
packages : Dict[str, str], default: {}
|
1658
|
-
Packages to use for this flow. The key is the name of the package
|
1659
|
-
and the value is the version to use.
|
1660
|
-
python : str, optional, default: None
|
1661
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1662
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1663
|
-
"""
|
1664
|
-
...
|
1665
|
-
|
1666
|
-
@typing.overload
|
1667
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1668
|
-
"""
|
1669
|
-
Specifies the Conda environment for all steps of the flow.
|
1670
|
-
|
1671
|
-
Use `@conda_base` to set common libraries required by all
|
1672
|
-
steps and use `@conda` to specify step-specific additions.
|
1673
|
-
|
1674
|
-
Parameters
|
1675
|
-
----------
|
1676
|
-
packages : Dict[str, str], default {}
|
1677
|
-
Packages to use for this flow. The key is the name of the package
|
1678
|
-
and the value is the version to use.
|
1679
|
-
libraries : Dict[str, str], default {}
|
1680
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1681
|
-
python : str, optional, default None
|
1682
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1683
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1684
|
-
disabled : bool, default False
|
1685
|
-
If set to True, disables Conda.
|
1686
|
-
"""
|
1687
|
-
...
|
1688
|
-
|
1689
|
-
@typing.overload
|
1690
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1691
|
-
...
|
1692
|
-
|
1693
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1648
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1694
1649
|
"""
|
1695
|
-
|
1696
|
-
|
1697
|
-
|
1698
|
-
|
1650
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1651
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1652
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1653
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1654
|
+
starts only after all sensors finish.
|
1699
1655
|
|
1700
1656
|
Parameters
|
1701
1657
|
----------
|
1702
|
-
|
1703
|
-
|
1704
|
-
|
1705
|
-
|
1706
|
-
|
1707
|
-
|
1708
|
-
|
1709
|
-
|
1710
|
-
|
1711
|
-
|
1658
|
+
timeout : int
|
1659
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1660
|
+
poke_interval : int
|
1661
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1662
|
+
mode : str
|
1663
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1664
|
+
exponential_backoff : bool
|
1665
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1666
|
+
pool : str
|
1667
|
+
the slot pool this task should run in,
|
1668
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1669
|
+
soft_fail : bool
|
1670
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1671
|
+
name : str
|
1672
|
+
Name of the sensor on Airflow
|
1673
|
+
description : str
|
1674
|
+
Description of sensor in the Airflow UI
|
1675
|
+
bucket_key : Union[str, List[str]]
|
1676
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1677
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1678
|
+
bucket_name : str
|
1679
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1680
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1681
|
+
wildcard_match : bool
|
1682
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1683
|
+
aws_conn_id : str
|
1684
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1685
|
+
verify : bool
|
1686
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1712
1687
|
"""
|
1713
1688
|
...
|
1714
1689
|
|
@@ -1807,45 +1782,70 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1807
1782
|
"""
|
1808
1783
|
...
|
1809
1784
|
|
1810
|
-
def
|
1785
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1811
1786
|
"""
|
1812
|
-
|
1813
|
-
|
1787
|
+
Specifies what flows belong to the same project.
|
1788
|
+
|
1789
|
+
A project-specific namespace is created for all flows that
|
1790
|
+
use the same `@project(name)`.
|
1814
1791
|
|
1815
1792
|
Parameters
|
1816
1793
|
----------
|
1817
|
-
timeout : int
|
1818
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1819
|
-
poke_interval : int
|
1820
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1821
|
-
mode : str
|
1822
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1823
|
-
exponential_backoff : bool
|
1824
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1825
|
-
pool : str
|
1826
|
-
the slot pool this task should run in,
|
1827
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1828
|
-
soft_fail : bool
|
1829
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1830
1794
|
name : str
|
1831
|
-
|
1832
|
-
|
1833
|
-
|
1834
|
-
|
1835
|
-
|
1836
|
-
|
1837
|
-
|
1838
|
-
|
1839
|
-
|
1840
|
-
|
1841
|
-
|
1842
|
-
|
1843
|
-
|
1844
|
-
|
1845
|
-
|
1846
|
-
|
1847
|
-
|
1848
|
-
|
1795
|
+
Project name. Make sure that the name is unique amongst all
|
1796
|
+
projects that use the same production scheduler. The name may
|
1797
|
+
contain only lowercase alphanumeric characters and underscores.
|
1798
|
+
|
1799
|
+
|
1800
|
+
"""
|
1801
|
+
...
|
1802
|
+
|
1803
|
+
@typing.overload
|
1804
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1805
|
+
"""
|
1806
|
+
Specifies the Conda environment for all steps of the flow.
|
1807
|
+
|
1808
|
+
Use `@conda_base` to set common libraries required by all
|
1809
|
+
steps and use `@conda` to specify step-specific additions.
|
1810
|
+
|
1811
|
+
Parameters
|
1812
|
+
----------
|
1813
|
+
packages : Dict[str, str], default {}
|
1814
|
+
Packages to use for this flow. The key is the name of the package
|
1815
|
+
and the value is the version to use.
|
1816
|
+
libraries : Dict[str, str], default {}
|
1817
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1818
|
+
python : str, optional, default None
|
1819
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1820
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1821
|
+
disabled : bool, default False
|
1822
|
+
If set to True, disables Conda.
|
1823
|
+
"""
|
1824
|
+
...
|
1825
|
+
|
1826
|
+
@typing.overload
|
1827
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1828
|
+
...
|
1829
|
+
|
1830
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1831
|
+
"""
|
1832
|
+
Specifies the Conda environment for all steps of the flow.
|
1833
|
+
|
1834
|
+
Use `@conda_base` to set common libraries required by all
|
1835
|
+
steps and use `@conda` to specify step-specific additions.
|
1836
|
+
|
1837
|
+
Parameters
|
1838
|
+
----------
|
1839
|
+
packages : Dict[str, str], default {}
|
1840
|
+
Packages to use for this flow. The key is the name of the package
|
1841
|
+
and the value is the version to use.
|
1842
|
+
libraries : Dict[str, str], default {}
|
1843
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1844
|
+
python : str, optional, default None
|
1845
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1846
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1847
|
+
disabled : bool, default False
|
1848
|
+
If set to True, disables Conda.
|
1849
1849
|
"""
|
1850
1850
|
...
|
1851
1851
|
|