metaflow-stubs 2.11.9__py2.py3-none-any.whl → 2.11.11__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +468 -466
- metaflow-stubs/cards.pyi +6 -6
- metaflow-stubs/cli.pyi +3 -3
- metaflow-stubs/client/__init__.pyi +3 -3
- metaflow-stubs/client/core.pyi +7 -7
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +5 -5
- metaflow-stubs/metadata/metadata.pyi +2 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +6 -2
- metaflow-stubs/metaflow_current.pyi +4 -4
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +5 -5
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +4 -4
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +4 -4
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +4 -4
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +4 -4
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +6 -4
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +5 -5
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.11.9.dist-info → metaflow_stubs-2.11.11.dist-info}/METADATA +2 -2
- metaflow_stubs-2.11.11.dist-info/RECORD +132 -0
- metaflow_stubs-2.11.9.dist-info/RECORD +0 -132
- {metaflow_stubs-2.11.9.dist-info → metaflow_stubs-2.11.11.dist-info}/WHEEL +0 -0
- {metaflow_stubs-2.11.9.dist-info → metaflow_stubs-2.11.11.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,23 +1,23 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.11.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.11.11 #
|
4
|
+
# Generated on 2024-05-02T22:04:13.800222 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.
|
11
|
+
import metaflow.client.core
|
12
|
+
import datetime
|
13
|
+
import metaflow._vendor.click.types
|
14
|
+
import typing
|
12
15
|
import metaflow.plugins.datatools.s3.s3
|
16
|
+
import metaflow.datastore.inputs
|
13
17
|
import metaflow.metaflow_current
|
14
|
-
import
|
18
|
+
import metaflow.parameters
|
15
19
|
import io
|
16
20
|
import metaflow.events
|
17
|
-
import metaflow.parameters
|
18
|
-
import datetime
|
19
|
-
import metaflow.client.core
|
20
|
-
import metaflow._vendor.click.types
|
21
21
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
22
22
|
StepFlag = typing.NewType("StepFlag", bool)
|
23
23
|
|
@@ -726,194 +726,51 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
726
726
|
...
|
727
727
|
|
728
728
|
@typing.overload
|
729
|
-
def
|
730
|
-
"""
|
731
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
732
|
-
|
733
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
734
|
-
|
735
|
-
Parameters
|
736
|
-
----------
|
737
|
-
type : str, default 'default'
|
738
|
-
Card type.
|
739
|
-
id : str, optional, default None
|
740
|
-
If multiple cards are present, use this id to identify this card.
|
741
|
-
options : Dict[str, Any], default {}
|
742
|
-
Options passed to the card. The contents depend on the card type.
|
743
|
-
timeout : int, default 45
|
744
|
-
Interrupt reporting if it takes more than this many seconds.
|
745
|
-
|
746
|
-
|
747
|
-
"""
|
748
|
-
...
|
749
|
-
|
750
|
-
@typing.overload
|
751
|
-
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
752
|
-
...
|
753
|
-
|
754
|
-
@typing.overload
|
755
|
-
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
756
|
-
...
|
757
|
-
|
758
|
-
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
759
|
-
"""
|
760
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
761
|
-
|
762
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
763
|
-
|
764
|
-
Parameters
|
765
|
-
----------
|
766
|
-
type : str, default 'default'
|
767
|
-
Card type.
|
768
|
-
id : str, optional, default None
|
769
|
-
If multiple cards are present, use this id to identify this card.
|
770
|
-
options : Dict[str, Any], default {}
|
771
|
-
Options passed to the card. The contents depend on the card type.
|
772
|
-
timeout : int, default 45
|
773
|
-
Interrupt reporting if it takes more than this many seconds.
|
774
|
-
|
775
|
-
|
776
|
-
"""
|
777
|
-
...
|
778
|
-
|
779
|
-
@typing.overload
|
780
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
781
|
-
"""
|
782
|
-
Specifies environment variables to be set prior to the execution of a step.
|
783
|
-
|
784
|
-
Parameters
|
785
|
-
----------
|
786
|
-
vars : Dict[str, str], default {}
|
787
|
-
Dictionary of environment variables to set.
|
788
|
-
"""
|
789
|
-
...
|
790
|
-
|
791
|
-
@typing.overload
|
792
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
793
|
-
...
|
794
|
-
|
795
|
-
@typing.overload
|
796
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
797
|
-
...
|
798
|
-
|
799
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
800
|
-
"""
|
801
|
-
Specifies environment variables to be set prior to the execution of a step.
|
802
|
-
|
803
|
-
Parameters
|
804
|
-
----------
|
805
|
-
vars : Dict[str, str], default {}
|
806
|
-
Dictionary of environment variables to set.
|
807
|
-
"""
|
808
|
-
...
|
809
|
-
|
810
|
-
@typing.overload
|
811
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
812
|
-
"""
|
813
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
814
|
-
the execution of a step.
|
815
|
-
|
816
|
-
Parameters
|
817
|
-
----------
|
818
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
819
|
-
List of secret specs, defining how the secrets are to be retrieved
|
820
|
-
"""
|
821
|
-
...
|
822
|
-
|
823
|
-
@typing.overload
|
824
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
825
|
-
...
|
826
|
-
|
827
|
-
@typing.overload
|
828
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
829
|
-
...
|
830
|
-
|
831
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
832
|
-
"""
|
833
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
834
|
-
the execution of a step.
|
835
|
-
|
836
|
-
Parameters
|
837
|
-
----------
|
838
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
839
|
-
List of secret specs, defining how the secrets are to be retrieved
|
840
|
-
"""
|
841
|
-
...
|
842
|
-
|
843
|
-
@typing.overload
|
844
|
-
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
729
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
845
730
|
"""
|
846
|
-
Specifies the
|
847
|
-
|
848
|
-
Use `@resources` to specify the resource requirements
|
849
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
731
|
+
Specifies the PyPI packages for the step.
|
850
732
|
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
|
855
|
-
or
|
856
|
-
```
|
857
|
-
python myflow.py run --with kubernetes
|
858
|
-
```
|
859
|
-
which executes the flow on the desired system using the
|
860
|
-
requirements specified in `@resources`.
|
733
|
+
Information in this decorator will augment any
|
734
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
735
|
+
you can use `@pypi_base` to set packages required by all
|
736
|
+
steps and use `@pypi` to specify step-specific overrides.
|
861
737
|
|
862
738
|
Parameters
|
863
739
|
----------
|
864
|
-
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
memory : int, default 4096
|
871
|
-
Memory size (in MB) required for this step.
|
872
|
-
shared_memory : int, optional, default None
|
873
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
874
|
-
This parameter maps to the `--shm-size` option in Docker.
|
740
|
+
packages : Dict[str, str], default: {}
|
741
|
+
Packages to use for this step. The key is the name of the package
|
742
|
+
and the value is the version to use.
|
743
|
+
python : str, optional, default: None
|
744
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
745
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
875
746
|
"""
|
876
747
|
...
|
877
748
|
|
878
749
|
@typing.overload
|
879
|
-
def
|
750
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
880
751
|
...
|
881
752
|
|
882
753
|
@typing.overload
|
883
|
-
def
|
754
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
884
755
|
...
|
885
756
|
|
886
|
-
def
|
757
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
887
758
|
"""
|
888
|
-
Specifies the
|
889
|
-
|
890
|
-
Use `@resources` to specify the resource requirements
|
891
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
759
|
+
Specifies the PyPI packages for the step.
|
892
760
|
|
893
|
-
|
894
|
-
|
895
|
-
|
896
|
-
|
897
|
-
or
|
898
|
-
```
|
899
|
-
python myflow.py run --with kubernetes
|
900
|
-
```
|
901
|
-
which executes the flow on the desired system using the
|
902
|
-
requirements specified in `@resources`.
|
761
|
+
Information in this decorator will augment any
|
762
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
763
|
+
you can use `@pypi_base` to set packages required by all
|
764
|
+
steps and use `@pypi` to specify step-specific overrides.
|
903
765
|
|
904
766
|
Parameters
|
905
767
|
----------
|
906
|
-
|
907
|
-
|
908
|
-
|
909
|
-
|
910
|
-
|
911
|
-
|
912
|
-
memory : int, default 4096
|
913
|
-
Memory size (in MB) required for this step.
|
914
|
-
shared_memory : int, optional, default None
|
915
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
916
|
-
This parameter maps to the `--shm-size` option in Docker.
|
768
|
+
packages : Dict[str, str], default: {}
|
769
|
+
Packages to use for this step. The key is the name of the package
|
770
|
+
and the value is the version to use.
|
771
|
+
python : str, optional, default: None
|
772
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
773
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
917
774
|
"""
|
918
775
|
...
|
919
776
|
|
@@ -1065,51 +922,187 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1065
922
|
...
|
1066
923
|
|
1067
924
|
@typing.overload
|
1068
|
-
def
|
925
|
+
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1069
926
|
"""
|
1070
|
-
Specifies the
|
927
|
+
Specifies the resources needed when executing this step.
|
1071
928
|
|
1072
|
-
|
1073
|
-
|
1074
|
-
|
1075
|
-
|
929
|
+
Use `@resources` to specify the resource requirements
|
930
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
931
|
+
|
932
|
+
You can choose the compute layer on the command line by executing e.g.
|
933
|
+
```
|
934
|
+
python myflow.py run --with batch
|
935
|
+
```
|
936
|
+
or
|
937
|
+
```
|
938
|
+
python myflow.py run --with kubernetes
|
939
|
+
```
|
940
|
+
which executes the flow on the desired system using the
|
941
|
+
requirements specified in `@resources`.
|
1076
942
|
|
1077
943
|
Parameters
|
1078
944
|
----------
|
1079
|
-
|
1080
|
-
|
1081
|
-
|
1082
|
-
|
1083
|
-
|
1084
|
-
|
945
|
+
cpu : int, default 1
|
946
|
+
Number of CPUs required for this step.
|
947
|
+
gpu : int, default 0
|
948
|
+
Number of GPUs required for this step.
|
949
|
+
disk : int, optional, default None
|
950
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
951
|
+
memory : int, default 4096
|
952
|
+
Memory size (in MB) required for this step.
|
953
|
+
shared_memory : int, optional, default None
|
954
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
955
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1085
956
|
"""
|
1086
957
|
...
|
1087
958
|
|
1088
959
|
@typing.overload
|
1089
|
-
def
|
960
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1090
961
|
...
|
1091
962
|
|
1092
963
|
@typing.overload
|
1093
|
-
def
|
964
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1094
965
|
...
|
1095
966
|
|
1096
|
-
def
|
967
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
1097
968
|
"""
|
1098
|
-
Specifies the
|
969
|
+
Specifies the resources needed when executing this step.
|
1099
970
|
|
1100
|
-
|
1101
|
-
|
1102
|
-
|
1103
|
-
|
971
|
+
Use `@resources` to specify the resource requirements
|
972
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
973
|
+
|
974
|
+
You can choose the compute layer on the command line by executing e.g.
|
975
|
+
```
|
976
|
+
python myflow.py run --with batch
|
977
|
+
```
|
978
|
+
or
|
979
|
+
```
|
980
|
+
python myflow.py run --with kubernetes
|
981
|
+
```
|
982
|
+
which executes the flow on the desired system using the
|
983
|
+
requirements specified in `@resources`.
|
1104
984
|
|
1105
985
|
Parameters
|
1106
986
|
----------
|
1107
|
-
|
1108
|
-
|
1109
|
-
|
1110
|
-
|
1111
|
-
|
1112
|
-
|
987
|
+
cpu : int, default 1
|
988
|
+
Number of CPUs required for this step.
|
989
|
+
gpu : int, default 0
|
990
|
+
Number of GPUs required for this step.
|
991
|
+
disk : int, optional, default None
|
992
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
993
|
+
memory : int, default 4096
|
994
|
+
Memory size (in MB) required for this step.
|
995
|
+
shared_memory : int, optional, default None
|
996
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
997
|
+
This parameter maps to the `--shm-size` option in Docker.
|
998
|
+
"""
|
999
|
+
...
|
1000
|
+
|
1001
|
+
@typing.overload
|
1002
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1003
|
+
"""
|
1004
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1005
|
+
|
1006
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1007
|
+
|
1008
|
+
Parameters
|
1009
|
+
----------
|
1010
|
+
type : str, default 'default'
|
1011
|
+
Card type.
|
1012
|
+
id : str, optional, default None
|
1013
|
+
If multiple cards are present, use this id to identify this card.
|
1014
|
+
options : Dict[str, Any], default {}
|
1015
|
+
Options passed to the card. The contents depend on the card type.
|
1016
|
+
timeout : int, default 45
|
1017
|
+
Interrupt reporting if it takes more than this many seconds.
|
1018
|
+
|
1019
|
+
|
1020
|
+
"""
|
1021
|
+
...
|
1022
|
+
|
1023
|
+
@typing.overload
|
1024
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1025
|
+
...
|
1026
|
+
|
1027
|
+
@typing.overload
|
1028
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1029
|
+
...
|
1030
|
+
|
1031
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
1032
|
+
"""
|
1033
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1034
|
+
|
1035
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1036
|
+
|
1037
|
+
Parameters
|
1038
|
+
----------
|
1039
|
+
type : str, default 'default'
|
1040
|
+
Card type.
|
1041
|
+
id : str, optional, default None
|
1042
|
+
If multiple cards are present, use this id to identify this card.
|
1043
|
+
options : Dict[str, Any], default {}
|
1044
|
+
Options passed to the card. The contents depend on the card type.
|
1045
|
+
timeout : int, default 45
|
1046
|
+
Interrupt reporting if it takes more than this many seconds.
|
1047
|
+
|
1048
|
+
|
1049
|
+
"""
|
1050
|
+
...
|
1051
|
+
|
1052
|
+
@typing.overload
|
1053
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1054
|
+
"""
|
1055
|
+
Specifies a timeout for your step.
|
1056
|
+
|
1057
|
+
This decorator is useful if this step may hang indefinitely.
|
1058
|
+
|
1059
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1060
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1061
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1062
|
+
|
1063
|
+
Note that all the values specified in parameters are added together so if you specify
|
1064
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1065
|
+
|
1066
|
+
Parameters
|
1067
|
+
----------
|
1068
|
+
seconds : int, default 0
|
1069
|
+
Number of seconds to wait prior to timing out.
|
1070
|
+
minutes : int, default 0
|
1071
|
+
Number of minutes to wait prior to timing out.
|
1072
|
+
hours : int, default 0
|
1073
|
+
Number of hours to wait prior to timing out.
|
1074
|
+
"""
|
1075
|
+
...
|
1076
|
+
|
1077
|
+
@typing.overload
|
1078
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1079
|
+
...
|
1080
|
+
|
1081
|
+
@typing.overload
|
1082
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1083
|
+
...
|
1084
|
+
|
1085
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1086
|
+
"""
|
1087
|
+
Specifies a timeout for your step.
|
1088
|
+
|
1089
|
+
This decorator is useful if this step may hang indefinitely.
|
1090
|
+
|
1091
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1092
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1093
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1094
|
+
|
1095
|
+
Note that all the values specified in parameters are added together so if you specify
|
1096
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1097
|
+
|
1098
|
+
Parameters
|
1099
|
+
----------
|
1100
|
+
seconds : int, default 0
|
1101
|
+
Number of seconds to wait prior to timing out.
|
1102
|
+
minutes : int, default 0
|
1103
|
+
Number of minutes to wait prior to timing out.
|
1104
|
+
hours : int, default 0
|
1105
|
+
Number of hours to wait prior to timing out.
|
1113
1106
|
"""
|
1114
1107
|
...
|
1115
1108
|
|
@@ -1215,7 +1208,128 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1215
1208
|
"""
|
1216
1209
|
...
|
1217
1210
|
|
1218
|
-
|
1211
|
+
@typing.overload
|
1212
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1213
|
+
"""
|
1214
|
+
Specifies the Conda environment for the step.
|
1215
|
+
|
1216
|
+
Information in this decorator will augment any
|
1217
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1218
|
+
you can use `@conda_base` to set packages required by all
|
1219
|
+
steps and use `@conda` to specify step-specific overrides.
|
1220
|
+
|
1221
|
+
Parameters
|
1222
|
+
----------
|
1223
|
+
packages : Dict[str, str], default {}
|
1224
|
+
Packages to use for this step. The key is the name of the package
|
1225
|
+
and the value is the version to use.
|
1226
|
+
libraries : Dict[str, str], default {}
|
1227
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1228
|
+
python : str, optional, default None
|
1229
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1230
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1231
|
+
disabled : bool, default False
|
1232
|
+
If set to True, disables @conda.
|
1233
|
+
"""
|
1234
|
+
...
|
1235
|
+
|
1236
|
+
@typing.overload
|
1237
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1238
|
+
...
|
1239
|
+
|
1240
|
+
@typing.overload
|
1241
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1242
|
+
...
|
1243
|
+
|
1244
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1245
|
+
"""
|
1246
|
+
Specifies the Conda environment for the step.
|
1247
|
+
|
1248
|
+
Information in this decorator will augment any
|
1249
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1250
|
+
you can use `@conda_base` to set packages required by all
|
1251
|
+
steps and use `@conda` to specify step-specific overrides.
|
1252
|
+
|
1253
|
+
Parameters
|
1254
|
+
----------
|
1255
|
+
packages : Dict[str, str], default {}
|
1256
|
+
Packages to use for this step. The key is the name of the package
|
1257
|
+
and the value is the version to use.
|
1258
|
+
libraries : Dict[str, str], default {}
|
1259
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1260
|
+
python : str, optional, default None
|
1261
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1262
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1263
|
+
disabled : bool, default False
|
1264
|
+
If set to True, disables @conda.
|
1265
|
+
"""
|
1266
|
+
...
|
1267
|
+
|
1268
|
+
@typing.overload
|
1269
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1270
|
+
"""
|
1271
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1272
|
+
the execution of a step.
|
1273
|
+
|
1274
|
+
Parameters
|
1275
|
+
----------
|
1276
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1277
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1278
|
+
"""
|
1279
|
+
...
|
1280
|
+
|
1281
|
+
@typing.overload
|
1282
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1283
|
+
...
|
1284
|
+
|
1285
|
+
@typing.overload
|
1286
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1287
|
+
...
|
1288
|
+
|
1289
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
1290
|
+
"""
|
1291
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1292
|
+
the execution of a step.
|
1293
|
+
|
1294
|
+
Parameters
|
1295
|
+
----------
|
1296
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1297
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1298
|
+
"""
|
1299
|
+
...
|
1300
|
+
|
1301
|
+
@typing.overload
|
1302
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1303
|
+
"""
|
1304
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1305
|
+
|
1306
|
+
Parameters
|
1307
|
+
----------
|
1308
|
+
vars : Dict[str, str], default {}
|
1309
|
+
Dictionary of environment variables to set.
|
1310
|
+
"""
|
1311
|
+
...
|
1312
|
+
|
1313
|
+
@typing.overload
|
1314
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1315
|
+
...
|
1316
|
+
|
1317
|
+
@typing.overload
|
1318
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1319
|
+
...
|
1320
|
+
|
1321
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1322
|
+
"""
|
1323
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1324
|
+
|
1325
|
+
Parameters
|
1326
|
+
----------
|
1327
|
+
vars : Dict[str, str], default {}
|
1328
|
+
Dictionary of environment variables to set.
|
1329
|
+
"""
|
1330
|
+
...
|
1331
|
+
|
1332
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1219
1333
|
"""
|
1220
1334
|
Specifies that this step should execute on Kubernetes.
|
1221
1335
|
|
@@ -1269,138 +1383,156 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
1269
1383
|
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
1270
1384
|
shared_memory: int, optional
|
1271
1385
|
Shared memory size (in MiB) required for this step
|
1386
|
+
port: int, optional
|
1387
|
+
Port number to specify in the Kubernetes job object
|
1272
1388
|
"""
|
1273
1389
|
...
|
1274
1390
|
|
1275
1391
|
@typing.overload
|
1276
|
-
def
|
1392
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1277
1393
|
"""
|
1278
|
-
Specifies the
|
1279
|
-
|
1280
|
-
Information in this decorator will augment any
|
1281
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1282
|
-
you can use `@conda_base` to set packages required by all
|
1283
|
-
steps and use `@conda` to specify step-specific overrides.
|
1394
|
+
Specifies the PyPI packages for all steps of the flow.
|
1284
1395
|
|
1396
|
+
Use `@pypi_base` to set common packages required by all
|
1397
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1285
1398
|
Parameters
|
1286
1399
|
----------
|
1287
|
-
packages : Dict[str, str], default {}
|
1288
|
-
Packages to use for this
|
1400
|
+
packages : Dict[str, str], default: {}
|
1401
|
+
Packages to use for this flow. The key is the name of the package
|
1289
1402
|
and the value is the version to use.
|
1290
|
-
|
1291
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1292
|
-
python : str, optional, default None
|
1403
|
+
python : str, optional, default: None
|
1293
1404
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1294
1405
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1295
|
-
disabled : bool, default False
|
1296
|
-
If set to True, disables @conda.
|
1297
1406
|
"""
|
1298
1407
|
...
|
1299
1408
|
|
1300
1409
|
@typing.overload
|
1301
|
-
def
|
1302
|
-
...
|
1303
|
-
|
1304
|
-
@typing.overload
|
1305
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1410
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1306
1411
|
...
|
1307
1412
|
|
1308
|
-
def
|
1413
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1309
1414
|
"""
|
1310
|
-
Specifies the
|
1311
|
-
|
1312
|
-
Information in this decorator will augment any
|
1313
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1314
|
-
you can use `@conda_base` to set packages required by all
|
1315
|
-
steps and use `@conda` to specify step-specific overrides.
|
1415
|
+
Specifies the PyPI packages for all steps of the flow.
|
1316
1416
|
|
1417
|
+
Use `@pypi_base` to set common packages required by all
|
1418
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1317
1419
|
Parameters
|
1318
1420
|
----------
|
1319
|
-
packages : Dict[str, str], default {}
|
1320
|
-
Packages to use for this
|
1421
|
+
packages : Dict[str, str], default: {}
|
1422
|
+
Packages to use for this flow. The key is the name of the package
|
1321
1423
|
and the value is the version to use.
|
1322
|
-
|
1323
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1324
|
-
python : str, optional, default None
|
1424
|
+
python : str, optional, default: None
|
1325
1425
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1326
1426
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1327
|
-
disabled : bool, default False
|
1328
|
-
If set to True, disables @conda.
|
1329
1427
|
"""
|
1330
1428
|
...
|
1331
1429
|
|
1332
|
-
|
1333
|
-
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1430
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1334
1431
|
"""
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1341
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1342
|
-
|
1343
|
-
Note that all the values specified in parameters are added together so if you specify
|
1344
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1432
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1433
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1434
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1435
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1436
|
+
starts only after all sensors finish.
|
1345
1437
|
|
1346
1438
|
Parameters
|
1347
1439
|
----------
|
1348
|
-
|
1349
|
-
|
1350
|
-
|
1351
|
-
|
1352
|
-
|
1353
|
-
|
1440
|
+
timeout : int
|
1441
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1442
|
+
poke_interval : int
|
1443
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1444
|
+
mode : str
|
1445
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1446
|
+
exponential_backoff : bool
|
1447
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1448
|
+
pool : str
|
1449
|
+
the slot pool this task should run in,
|
1450
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1451
|
+
soft_fail : bool
|
1452
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1453
|
+
name : str
|
1454
|
+
Name of the sensor on Airflow
|
1455
|
+
description : str
|
1456
|
+
Description of sensor in the Airflow UI
|
1457
|
+
bucket_key : Union[str, List[str]]
|
1458
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1459
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1460
|
+
bucket_name : str
|
1461
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1462
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1463
|
+
wildcard_match : bool
|
1464
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1465
|
+
aws_conn_id : str
|
1466
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1467
|
+
verify : bool
|
1468
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1354
1469
|
"""
|
1355
1470
|
...
|
1356
1471
|
|
1357
|
-
|
1358
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1359
|
-
...
|
1360
|
-
|
1361
|
-
@typing.overload
|
1362
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1363
|
-
...
|
1364
|
-
|
1365
|
-
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1472
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1366
1473
|
"""
|
1367
|
-
Specifies
|
1474
|
+
Specifies what flows belong to the same project.
|
1368
1475
|
|
1369
|
-
|
1476
|
+
A project-specific namespace is created for all flows that
|
1477
|
+
use the same `@project(name)`.
|
1370
1478
|
|
1371
|
-
|
1372
|
-
|
1373
|
-
|
1479
|
+
Parameters
|
1480
|
+
----------
|
1481
|
+
name : str
|
1482
|
+
Project name. Make sure that the name is unique amongst all
|
1483
|
+
projects that use the same production scheduler. The name may
|
1484
|
+
contain only lowercase alphanumeric characters and underscores.
|
1374
1485
|
|
1375
|
-
|
1376
|
-
|
1486
|
+
|
1487
|
+
"""
|
1488
|
+
...
|
1489
|
+
|
1490
|
+
@typing.overload
|
1491
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1492
|
+
"""
|
1493
|
+
Specifies the times when the flow should be run when running on a
|
1494
|
+
production scheduler.
|
1377
1495
|
|
1378
1496
|
Parameters
|
1379
1497
|
----------
|
1380
|
-
|
1381
|
-
|
1382
|
-
|
1383
|
-
|
1384
|
-
|
1385
|
-
|
1498
|
+
hourly : bool, default False
|
1499
|
+
Run the workflow hourly.
|
1500
|
+
daily : bool, default True
|
1501
|
+
Run the workflow daily.
|
1502
|
+
weekly : bool, default False
|
1503
|
+
Run the workflow weekly.
|
1504
|
+
cron : str, optional, default None
|
1505
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1506
|
+
specified by this expression.
|
1507
|
+
timezone : str, optional, default None
|
1508
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1509
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1386
1510
|
"""
|
1387
1511
|
...
|
1388
1512
|
|
1389
|
-
|
1513
|
+
@typing.overload
|
1514
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1515
|
+
...
|
1516
|
+
|
1517
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1390
1518
|
"""
|
1391
|
-
Specifies
|
1392
|
-
|
1393
|
-
A project-specific namespace is created for all flows that
|
1394
|
-
use the same `@project(name)`.
|
1519
|
+
Specifies the times when the flow should be run when running on a
|
1520
|
+
production scheduler.
|
1395
1521
|
|
1396
1522
|
Parameters
|
1397
1523
|
----------
|
1398
|
-
|
1399
|
-
|
1400
|
-
|
1401
|
-
|
1402
|
-
|
1403
|
-
|
1524
|
+
hourly : bool, default False
|
1525
|
+
Run the workflow hourly.
|
1526
|
+
daily : bool, default True
|
1527
|
+
Run the workflow daily.
|
1528
|
+
weekly : bool, default False
|
1529
|
+
Run the workflow weekly.
|
1530
|
+
cron : str, optional, default None
|
1531
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1532
|
+
specified by this expression.
|
1533
|
+
timezone : str, optional, default None
|
1534
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1535
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1404
1536
|
"""
|
1405
1537
|
...
|
1406
1538
|
|
@@ -1499,52 +1631,10 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1499
1631
|
"""
|
1500
1632
|
...
|
1501
1633
|
|
1502
|
-
|
1503
|
-
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1504
|
-
"""
|
1505
|
-
Specifies the PyPI packages for all steps of the flow.
|
1506
|
-
|
1507
|
-
Use `@pypi_base` to set common packages required by all
|
1508
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1509
|
-
Parameters
|
1510
|
-
----------
|
1511
|
-
packages : Dict[str, str], default: {}
|
1512
|
-
Packages to use for this flow. The key is the name of the package
|
1513
|
-
and the value is the version to use.
|
1514
|
-
python : str, optional, default: None
|
1515
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1516
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1517
|
-
"""
|
1518
|
-
...
|
1519
|
-
|
1520
|
-
@typing.overload
|
1521
|
-
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1522
|
-
...
|
1523
|
-
|
1524
|
-
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1525
|
-
"""
|
1526
|
-
Specifies the PyPI packages for all steps of the flow.
|
1527
|
-
|
1528
|
-
Use `@pypi_base` to set common packages required by all
|
1529
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1530
|
-
Parameters
|
1531
|
-
----------
|
1532
|
-
packages : Dict[str, str], default: {}
|
1533
|
-
Packages to use for this flow. The key is the name of the package
|
1534
|
-
and the value is the version to use.
|
1535
|
-
python : str, optional, default: None
|
1536
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1537
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1538
|
-
"""
|
1539
|
-
...
|
1540
|
-
|
1541
|
-
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1634
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1542
1635
|
"""
|
1543
|
-
The `@
|
1544
|
-
|
1545
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1546
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1547
|
-
starts only after all sensors finish.
|
1636
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1637
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1548
1638
|
|
1549
1639
|
Parameters
|
1550
1640
|
----------
|
@@ -1565,67 +1655,21 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1565
1655
|
Name of the sensor on Airflow
|
1566
1656
|
description : str
|
1567
1657
|
Description of sensor in the Airflow UI
|
1568
|
-
|
1569
|
-
The
|
1570
|
-
|
1571
|
-
|
1572
|
-
|
1573
|
-
|
1574
|
-
|
1575
|
-
|
1576
|
-
|
1577
|
-
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
1581
|
-
|
1582
|
-
|
1583
|
-
@typing.overload
|
1584
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1585
|
-
"""
|
1586
|
-
Specifies the Conda environment for all steps of the flow.
|
1587
|
-
|
1588
|
-
Use `@conda_base` to set common libraries required by all
|
1589
|
-
steps and use `@conda` to specify step-specific additions.
|
1590
|
-
|
1591
|
-
Parameters
|
1592
|
-
----------
|
1593
|
-
packages : Dict[str, str], default {}
|
1594
|
-
Packages to use for this flow. The key is the name of the package
|
1595
|
-
and the value is the version to use.
|
1596
|
-
libraries : Dict[str, str], default {}
|
1597
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1598
|
-
python : str, optional, default None
|
1599
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1600
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1601
|
-
disabled : bool, default False
|
1602
|
-
If set to True, disables Conda.
|
1603
|
-
"""
|
1604
|
-
...
|
1605
|
-
|
1606
|
-
@typing.overload
|
1607
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1608
|
-
...
|
1609
|
-
|
1610
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1611
|
-
"""
|
1612
|
-
Specifies the Conda environment for all steps of the flow.
|
1613
|
-
|
1614
|
-
Use `@conda_base` to set common libraries required by all
|
1615
|
-
steps and use `@conda` to specify step-specific additions.
|
1616
|
-
|
1617
|
-
Parameters
|
1618
|
-
----------
|
1619
|
-
packages : Dict[str, str], default {}
|
1620
|
-
Packages to use for this flow. The key is the name of the package
|
1621
|
-
and the value is the version to use.
|
1622
|
-
libraries : Dict[str, str], default {}
|
1623
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1624
|
-
python : str, optional, default None
|
1625
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1626
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1627
|
-
disabled : bool, default False
|
1628
|
-
If set to True, disables Conda.
|
1658
|
+
external_dag_id : str
|
1659
|
+
The dag_id that contains the task you want to wait for.
|
1660
|
+
external_task_ids : List[str]
|
1661
|
+
The list of task_ids that you want to wait for.
|
1662
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1663
|
+
allowed_states : List[str]
|
1664
|
+
Iterable of allowed states, (Default: ['success'])
|
1665
|
+
failed_states : List[str]
|
1666
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1667
|
+
execution_delta : datetime.timedelta
|
1668
|
+
time difference with the previous execution to look at,
|
1669
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1670
|
+
check_existence: bool
|
1671
|
+
Set to True to check if the external task exists or check if
|
1672
|
+
the DAG to wait for exists. (Default: True)
|
1629
1673
|
"""
|
1630
1674
|
...
|
1631
1675
|
|
@@ -1733,93 +1777,51 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1733
1777
|
...
|
1734
1778
|
|
1735
1779
|
@typing.overload
|
1736
|
-
def
|
1780
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1737
1781
|
"""
|
1738
|
-
Specifies the
|
1739
|
-
|
1782
|
+
Specifies the Conda environment for all steps of the flow.
|
1783
|
+
|
1784
|
+
Use `@conda_base` to set common libraries required by all
|
1785
|
+
steps and use `@conda` to specify step-specific additions.
|
1740
1786
|
|
1741
1787
|
Parameters
|
1742
1788
|
----------
|
1743
|
-
|
1744
|
-
|
1745
|
-
|
1746
|
-
|
1747
|
-
|
1748
|
-
|
1749
|
-
|
1750
|
-
|
1751
|
-
|
1752
|
-
|
1753
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1754
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1789
|
+
packages : Dict[str, str], default {}
|
1790
|
+
Packages to use for this flow. The key is the name of the package
|
1791
|
+
and the value is the version to use.
|
1792
|
+
libraries : Dict[str, str], default {}
|
1793
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1794
|
+
python : str, optional, default None
|
1795
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1796
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1797
|
+
disabled : bool, default False
|
1798
|
+
If set to True, disables Conda.
|
1755
1799
|
"""
|
1756
1800
|
...
|
1757
1801
|
|
1758
1802
|
@typing.overload
|
1759
|
-
def
|
1803
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1760
1804
|
...
|
1761
1805
|
|
1762
|
-
def
|
1806
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1763
1807
|
"""
|
1764
|
-
Specifies the
|
1765
|
-
production scheduler.
|
1808
|
+
Specifies the Conda environment for all steps of the flow.
|
1766
1809
|
|
1767
|
-
|
1768
|
-
|
1769
|
-
hourly : bool, default False
|
1770
|
-
Run the workflow hourly.
|
1771
|
-
daily : bool, default True
|
1772
|
-
Run the workflow daily.
|
1773
|
-
weekly : bool, default False
|
1774
|
-
Run the workflow weekly.
|
1775
|
-
cron : str, optional, default None
|
1776
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1777
|
-
specified by this expression.
|
1778
|
-
timezone : str, optional, default None
|
1779
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1780
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1781
|
-
"""
|
1782
|
-
...
|
1783
|
-
|
1784
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1785
|
-
"""
|
1786
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1787
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1810
|
+
Use `@conda_base` to set common libraries required by all
|
1811
|
+
steps and use `@conda` to specify step-specific additions.
|
1788
1812
|
|
1789
1813
|
Parameters
|
1790
1814
|
----------
|
1791
|
-
|
1792
|
-
|
1793
|
-
|
1794
|
-
|
1795
|
-
|
1796
|
-
|
1797
|
-
|
1798
|
-
|
1799
|
-
|
1800
|
-
|
1801
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1802
|
-
soft_fail : bool
|
1803
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1804
|
-
name : str
|
1805
|
-
Name of the sensor on Airflow
|
1806
|
-
description : str
|
1807
|
-
Description of sensor in the Airflow UI
|
1808
|
-
external_dag_id : str
|
1809
|
-
The dag_id that contains the task you want to wait for.
|
1810
|
-
external_task_ids : List[str]
|
1811
|
-
The list of task_ids that you want to wait for.
|
1812
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
1813
|
-
allowed_states : List[str]
|
1814
|
-
Iterable of allowed states, (Default: ['success'])
|
1815
|
-
failed_states : List[str]
|
1816
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
1817
|
-
execution_delta : datetime.timedelta
|
1818
|
-
time difference with the previous execution to look at,
|
1819
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
1820
|
-
check_existence: bool
|
1821
|
-
Set to True to check if the external task exists or check if
|
1822
|
-
the DAG to wait for exists. (Default: True)
|
1815
|
+
packages : Dict[str, str], default {}
|
1816
|
+
Packages to use for this flow. The key is the name of the package
|
1817
|
+
and the value is the version to use.
|
1818
|
+
libraries : Dict[str, str], default {}
|
1819
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1820
|
+
python : str, optional, default None
|
1821
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1822
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1823
|
+
disabled : bool, default False
|
1824
|
+
If set to True, disables Conda.
|
1823
1825
|
"""
|
1824
1826
|
...
|
1825
1827
|
|