metaflow-stubs 2.12.5__py2.py3-none-any.whl → 2.12.6__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +620 -620
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +4 -4
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +3 -3
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +4 -4
- metaflow-stubs/metadata/metadata.pyi +2 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +5 -5
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -2
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +3 -3
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
- metaflow-stubs/plugins/cards/card_client.pyi +2 -2
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +6 -6
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +4 -4
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +4 -4
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/metaflow_runner.pyi +5 -5
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.6.dist-info}/METADATA +2 -2
- metaflow_stubs-2.12.6.dist-info/RECORD +142 -0
- {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.6.dist-info}/WHEEL +1 -1
- metaflow_stubs-2.12.5.dist-info/RECORD +0 -142
- {metaflow_stubs-2.12.5.dist-info → metaflow_stubs-2.12.6.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,25 +1,25 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-
|
3
|
+
# MF version: 2.12.6 #
|
4
|
+
# Generated on 2024-07-01T22:37:35.546266 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.client.core
|
12
|
-
import metaflow.runner.metaflow_runner
|
13
|
-
import metaflow._vendor.click.types
|
14
|
-
import typing
|
15
11
|
import metaflow.parameters
|
16
12
|
import metaflow.flowspec
|
17
|
-
import metaflow.plugins.datatools.s3.s3
|
18
13
|
import datetime
|
19
|
-
import io
|
20
|
-
import metaflow.metaflow_current
|
21
14
|
import metaflow.datastore.inputs
|
15
|
+
import metaflow.plugins.datatools.s3.s3
|
16
|
+
import metaflow.client.core
|
22
17
|
import metaflow.events
|
18
|
+
import metaflow.runner.metaflow_runner
|
19
|
+
import metaflow._vendor.click.types
|
20
|
+
import typing
|
21
|
+
import metaflow.metaflow_current
|
22
|
+
import io
|
23
23
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
24
24
|
StepFlag = typing.NewType("StepFlag", bool)
|
25
25
|
|
@@ -728,182 +728,145 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
728
728
|
...
|
729
729
|
|
730
730
|
@typing.overload
|
731
|
-
def
|
731
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
732
732
|
"""
|
733
|
-
Specifies
|
734
|
-
|
733
|
+
Specifies the number of times the task corresponding
|
734
|
+
to a step needs to be retried.
|
735
|
+
|
736
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
737
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
738
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
739
|
+
|
740
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
741
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
742
|
+
ensuring that the flow execution can continue.
|
735
743
|
|
736
744
|
Parameters
|
737
745
|
----------
|
738
|
-
|
739
|
-
|
746
|
+
times : int, default 3
|
747
|
+
Number of times to retry this task.
|
748
|
+
minutes_between_retries : int, default 2
|
749
|
+
Number of minutes between retries.
|
740
750
|
"""
|
741
751
|
...
|
742
752
|
|
743
753
|
@typing.overload
|
744
|
-
def
|
754
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
745
755
|
...
|
746
756
|
|
747
757
|
@typing.overload
|
748
|
-
def
|
758
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
749
759
|
...
|
750
760
|
|
751
|
-
def
|
761
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
752
762
|
"""
|
753
|
-
Specifies
|
754
|
-
|
763
|
+
Specifies the number of times the task corresponding
|
764
|
+
to a step needs to be retried.
|
765
|
+
|
766
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
767
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
768
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
769
|
+
|
770
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
771
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
772
|
+
ensuring that the flow execution can continue.
|
755
773
|
|
756
774
|
Parameters
|
757
775
|
----------
|
758
|
-
|
759
|
-
|
776
|
+
times : int, default 3
|
777
|
+
Number of times to retry this task.
|
778
|
+
minutes_between_retries : int, default 2
|
779
|
+
Number of minutes between retries.
|
760
780
|
"""
|
761
781
|
...
|
762
782
|
|
763
783
|
@typing.overload
|
764
|
-
def
|
784
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
765
785
|
"""
|
766
|
-
Specifies
|
786
|
+
Specifies the Conda environment for the step.
|
787
|
+
|
788
|
+
Information in this decorator will augment any
|
789
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
790
|
+
you can use `@conda_base` to set packages required by all
|
791
|
+
steps and use `@conda` to specify step-specific overrides.
|
767
792
|
|
768
793
|
Parameters
|
769
794
|
----------
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
|
780
|
-
image : str, optional, default None
|
781
|
-
Docker image to use when launching on AWS Batch. If not specified, and
|
782
|
-
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
783
|
-
not, a default Docker image mapping to the current version of Python is used.
|
784
|
-
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
785
|
-
AWS Batch Job Queue to submit the job to.
|
786
|
-
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
787
|
-
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
788
|
-
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
789
|
-
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
790
|
-
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
791
|
-
shared_memory : int, optional, default None
|
792
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
793
|
-
This parameter maps to the `--shm-size` option in Docker.
|
794
|
-
max_swap : int, optional, default None
|
795
|
-
The total amount of swap memory (in MiB) a container can use for this
|
796
|
-
step. This parameter is translated to the `--memory-swap` option in
|
797
|
-
Docker where the value is the sum of the container memory plus the
|
798
|
-
`max_swap` value.
|
799
|
-
swappiness : int, optional, default None
|
800
|
-
This allows you to tune memory swappiness behavior for this step.
|
801
|
-
A swappiness value of 0 causes swapping not to happen unless absolutely
|
802
|
-
necessary. A swappiness value of 100 causes pages to be swapped very
|
803
|
-
aggressively. Accepted values are whole numbers between 0 and 100.
|
804
|
-
use_tmpfs : bool, default False
|
805
|
-
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
806
|
-
not available on Fargate compute environments
|
807
|
-
tmpfs_tempdir : bool, default True
|
808
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
809
|
-
tmpfs_size : int, optional, default None
|
810
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
811
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
812
|
-
memory allocated for this step.
|
813
|
-
tmpfs_path : str, optional, default None
|
814
|
-
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
815
|
-
inferentia : int, default 0
|
816
|
-
Number of Inferentia chips required for this step.
|
817
|
-
trainium : int, default None
|
818
|
-
Alias for inferentia. Use only one of the two.
|
819
|
-
efa : int, default 0
|
820
|
-
Number of elastic fabric adapter network devices to attach to container
|
821
|
-
ephemeral_storage : int, default None
|
822
|
-
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
823
|
-
This is only relevant for Fargate compute environments
|
824
|
-
log_driver: str, optional, default None
|
825
|
-
The log driver to use for the Amazon ECS container.
|
826
|
-
log_options: List[str], optional, default None
|
827
|
-
List of strings containing options for the chosen log driver. The configurable values
|
828
|
-
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
829
|
-
Example: [`awslogs-group:aws/batch/job`]
|
795
|
+
packages : Dict[str, str], default {}
|
796
|
+
Packages to use for this step. The key is the name of the package
|
797
|
+
and the value is the version to use.
|
798
|
+
libraries : Dict[str, str], default {}
|
799
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
800
|
+
python : str, optional, default None
|
801
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
802
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
803
|
+
disabled : bool, default False
|
804
|
+
If set to True, disables @conda.
|
830
805
|
"""
|
831
806
|
...
|
832
807
|
|
833
808
|
@typing.overload
|
834
|
-
def
|
809
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
835
810
|
...
|
836
811
|
|
837
812
|
@typing.overload
|
838
|
-
def
|
813
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
839
814
|
...
|
840
815
|
|
841
|
-
def
|
816
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
842
817
|
"""
|
843
|
-
Specifies
|
818
|
+
Specifies the Conda environment for the step.
|
819
|
+
|
820
|
+
Information in this decorator will augment any
|
821
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
822
|
+
you can use `@conda_base` to set packages required by all
|
823
|
+
steps and use `@conda` to specify step-specific overrides.
|
844
824
|
|
845
825
|
Parameters
|
846
826
|
----------
|
847
|
-
|
848
|
-
|
849
|
-
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
|
855
|
-
|
856
|
-
|
857
|
-
|
858
|
-
|
859
|
-
|
860
|
-
|
861
|
-
|
862
|
-
|
863
|
-
|
864
|
-
|
865
|
-
|
866
|
-
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
|
875
|
-
|
876
|
-
|
877
|
-
|
878
|
-
|
879
|
-
|
880
|
-
|
881
|
-
|
882
|
-
|
883
|
-
|
884
|
-
|
885
|
-
|
886
|
-
|
887
|
-
|
888
|
-
|
889
|
-
|
890
|
-
tmpfs_path : str, optional, default None
|
891
|
-
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
892
|
-
inferentia : int, default 0
|
893
|
-
Number of Inferentia chips required for this step.
|
894
|
-
trainium : int, default None
|
895
|
-
Alias for inferentia. Use only one of the two.
|
896
|
-
efa : int, default 0
|
897
|
-
Number of elastic fabric adapter network devices to attach to container
|
898
|
-
ephemeral_storage : int, default None
|
899
|
-
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
900
|
-
This is only relevant for Fargate compute environments
|
901
|
-
log_driver: str, optional, default None
|
902
|
-
The log driver to use for the Amazon ECS container.
|
903
|
-
log_options: List[str], optional, default None
|
904
|
-
List of strings containing options for the chosen log driver. The configurable values
|
905
|
-
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
906
|
-
Example: [`awslogs-group:aws/batch/job`]
|
827
|
+
packages : Dict[str, str], default {}
|
828
|
+
Packages to use for this step. The key is the name of the package
|
829
|
+
and the value is the version to use.
|
830
|
+
libraries : Dict[str, str], default {}
|
831
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
832
|
+
python : str, optional, default None
|
833
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
834
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
835
|
+
disabled : bool, default False
|
836
|
+
If set to True, disables @conda.
|
837
|
+
"""
|
838
|
+
...
|
839
|
+
|
840
|
+
@typing.overload
|
841
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
842
|
+
"""
|
843
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
844
|
+
the execution of a step.
|
845
|
+
|
846
|
+
Parameters
|
847
|
+
----------
|
848
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
849
|
+
List of secret specs, defining how the secrets are to be retrieved
|
850
|
+
"""
|
851
|
+
...
|
852
|
+
|
853
|
+
@typing.overload
|
854
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
855
|
+
...
|
856
|
+
|
857
|
+
@typing.overload
|
858
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
859
|
+
...
|
860
|
+
|
861
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
862
|
+
"""
|
863
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
864
|
+
the execution of a step.
|
865
|
+
|
866
|
+
Parameters
|
867
|
+
----------
|
868
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
869
|
+
List of secret specs, defining how the secrets are to be retrieved
|
907
870
|
"""
|
908
871
|
...
|
909
872
|
|
@@ -967,243 +930,59 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
967
930
|
...
|
968
931
|
|
969
932
|
@typing.overload
|
970
|
-
def
|
933
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
971
934
|
"""
|
972
|
-
Specifies
|
973
|
-
|
974
|
-
Information in this decorator will augment any
|
975
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
976
|
-
you can use `@conda_base` to set packages required by all
|
977
|
-
steps and use `@conda` to specify step-specific overrides.
|
978
|
-
|
979
|
-
Parameters
|
980
|
-
----------
|
981
|
-
packages : Dict[str, str], default {}
|
982
|
-
Packages to use for this step. The key is the name of the package
|
983
|
-
and the value is the version to use.
|
984
|
-
libraries : Dict[str, str], default {}
|
985
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
986
|
-
python : str, optional, default None
|
987
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
988
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
989
|
-
disabled : bool, default False
|
990
|
-
If set to True, disables @conda.
|
991
|
-
"""
|
992
|
-
...
|
993
|
-
|
994
|
-
@typing.overload
|
995
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
996
|
-
...
|
997
|
-
|
998
|
-
@typing.overload
|
999
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1000
|
-
...
|
1001
|
-
|
1002
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1003
|
-
"""
|
1004
|
-
Specifies the Conda environment for the step.
|
1005
|
-
|
1006
|
-
Information in this decorator will augment any
|
1007
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1008
|
-
you can use `@conda_base` to set packages required by all
|
1009
|
-
steps and use `@conda` to specify step-specific overrides.
|
1010
|
-
|
1011
|
-
Parameters
|
1012
|
-
----------
|
1013
|
-
packages : Dict[str, str], default {}
|
1014
|
-
Packages to use for this step. The key is the name of the package
|
1015
|
-
and the value is the version to use.
|
1016
|
-
libraries : Dict[str, str], default {}
|
1017
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1018
|
-
python : str, optional, default None
|
1019
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1020
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1021
|
-
disabled : bool, default False
|
1022
|
-
If set to True, disables @conda.
|
1023
|
-
"""
|
1024
|
-
...
|
1025
|
-
|
1026
|
-
@typing.overload
|
1027
|
-
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1028
|
-
"""
|
1029
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1030
|
-
|
1031
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1032
|
-
|
1033
|
-
Parameters
|
1034
|
-
----------
|
1035
|
-
type : str, default 'default'
|
1036
|
-
Card type.
|
1037
|
-
id : str, optional, default None
|
1038
|
-
If multiple cards are present, use this id to identify this card.
|
1039
|
-
options : Dict[str, Any], default {}
|
1040
|
-
Options passed to the card. The contents depend on the card type.
|
1041
|
-
timeout : int, default 45
|
1042
|
-
Interrupt reporting if it takes more than this many seconds.
|
1043
|
-
|
1044
|
-
|
1045
|
-
"""
|
1046
|
-
...
|
1047
|
-
|
1048
|
-
@typing.overload
|
1049
|
-
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1050
|
-
...
|
1051
|
-
|
1052
|
-
@typing.overload
|
1053
|
-
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1054
|
-
...
|
1055
|
-
|
1056
|
-
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
1057
|
-
"""
|
1058
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1059
|
-
|
1060
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1061
|
-
|
1062
|
-
Parameters
|
1063
|
-
----------
|
1064
|
-
type : str, default 'default'
|
1065
|
-
Card type.
|
1066
|
-
id : str, optional, default None
|
1067
|
-
If multiple cards are present, use this id to identify this card.
|
1068
|
-
options : Dict[str, Any], default {}
|
1069
|
-
Options passed to the card. The contents depend on the card type.
|
1070
|
-
timeout : int, default 45
|
1071
|
-
Interrupt reporting if it takes more than this many seconds.
|
1072
|
-
|
1073
|
-
|
1074
|
-
"""
|
1075
|
-
...
|
1076
|
-
|
1077
|
-
@typing.overload
|
1078
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1079
|
-
"""
|
1080
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1081
|
-
|
1082
|
-
Parameters
|
1083
|
-
----------
|
1084
|
-
vars : Dict[str, str], default {}
|
1085
|
-
Dictionary of environment variables to set.
|
1086
|
-
"""
|
1087
|
-
...
|
1088
|
-
|
1089
|
-
@typing.overload
|
1090
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1091
|
-
...
|
1092
|
-
|
1093
|
-
@typing.overload
|
1094
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1095
|
-
...
|
1096
|
-
|
1097
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1098
|
-
"""
|
1099
|
-
Specifies environment variables to be set prior to the execution of a step.
|
935
|
+
Specifies a timeout for your step.
|
1100
936
|
|
1101
|
-
|
1102
|
-
----------
|
1103
|
-
vars : Dict[str, str], default {}
|
1104
|
-
Dictionary of environment variables to set.
|
1105
|
-
"""
|
1106
|
-
...
|
1107
|
-
|
1108
|
-
@typing.overload
|
1109
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1110
|
-
"""
|
1111
|
-
Specifies the number of times the task corresponding
|
1112
|
-
to a step needs to be retried.
|
937
|
+
This decorator is useful if this step may hang indefinitely.
|
1113
938
|
|
1114
|
-
This
|
1115
|
-
|
1116
|
-
|
939
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
940
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
941
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1117
942
|
|
1118
|
-
|
1119
|
-
decorator will
|
1120
|
-
ensuring that the flow execution can continue.
|
943
|
+
Note that all the values specified in parameters are added together so if you specify
|
944
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1121
945
|
|
1122
946
|
Parameters
|
1123
947
|
----------
|
1124
|
-
|
1125
|
-
Number of
|
1126
|
-
|
1127
|
-
Number of minutes
|
948
|
+
seconds : int, default 0
|
949
|
+
Number of seconds to wait prior to timing out.
|
950
|
+
minutes : int, default 0
|
951
|
+
Number of minutes to wait prior to timing out.
|
952
|
+
hours : int, default 0
|
953
|
+
Number of hours to wait prior to timing out.
|
1128
954
|
"""
|
1129
955
|
...
|
1130
956
|
|
1131
957
|
@typing.overload
|
1132
|
-
def
|
958
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1133
959
|
...
|
1134
960
|
|
1135
961
|
@typing.overload
|
1136
|
-
def
|
1137
|
-
...
|
1138
|
-
|
1139
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
1140
|
-
"""
|
1141
|
-
Specifies the number of times the task corresponding
|
1142
|
-
to a step needs to be retried.
|
1143
|
-
|
1144
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
1145
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
1146
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
1147
|
-
|
1148
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
1149
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
1150
|
-
ensuring that the flow execution can continue.
|
1151
|
-
|
1152
|
-
Parameters
|
1153
|
-
----------
|
1154
|
-
times : int, default 3
|
1155
|
-
Number of times to retry this task.
|
1156
|
-
minutes_between_retries : int, default 2
|
1157
|
-
Number of minutes between retries.
|
1158
|
-
"""
|
962
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1159
963
|
...
|
1160
964
|
|
1161
|
-
|
1162
|
-
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
965
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1163
966
|
"""
|
1164
|
-
Specifies
|
967
|
+
Specifies a timeout for your step.
|
1165
968
|
|
1166
|
-
|
1167
|
-
contains the exception raised. You can use it to detect the presence
|
1168
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
1169
|
-
are missing.
|
969
|
+
This decorator is useful if this step may hang indefinitely.
|
1170
970
|
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
Name of the artifact in which to store the caught exception.
|
1175
|
-
If not specified, the exception is not stored.
|
1176
|
-
print_exception : bool, default True
|
1177
|
-
Determines whether or not the exception is printed to
|
1178
|
-
stdout when caught.
|
1179
|
-
"""
|
1180
|
-
...
|
1181
|
-
|
1182
|
-
@typing.overload
|
1183
|
-
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1184
|
-
...
|
1185
|
-
|
1186
|
-
@typing.overload
|
1187
|
-
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1188
|
-
...
|
1189
|
-
|
1190
|
-
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
1191
|
-
"""
|
1192
|
-
Specifies that the step will success under all circumstances.
|
971
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
972
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
973
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1193
974
|
|
1194
|
-
|
1195
|
-
|
1196
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
1197
|
-
are missing.
|
975
|
+
Note that all the values specified in parameters are added together so if you specify
|
976
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1198
977
|
|
1199
978
|
Parameters
|
1200
979
|
----------
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
980
|
+
seconds : int, default 0
|
981
|
+
Number of seconds to wait prior to timing out.
|
982
|
+
minutes : int, default 0
|
983
|
+
Number of minutes to wait prior to timing out.
|
984
|
+
hours : int, default 0
|
985
|
+
Number of hours to wait prior to timing out.
|
1207
986
|
"""
|
1208
987
|
...
|
1209
988
|
|
@@ -1285,108 +1064,450 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
1285
1064
|
...
|
1286
1065
|
|
1287
1066
|
@typing.overload
|
1288
|
-
def
|
1067
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1289
1068
|
"""
|
1290
|
-
Specifies
|
1291
|
-
|
1292
|
-
This decorator is useful if this step may hang indefinitely.
|
1069
|
+
Specifies the PyPI packages for the step.
|
1293
1070
|
|
1294
|
-
|
1295
|
-
|
1296
|
-
|
1071
|
+
Information in this decorator will augment any
|
1072
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1073
|
+
you can use `@pypi_base` to set packages required by all
|
1074
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1297
1075
|
|
1298
|
-
|
1299
|
-
|
1076
|
+
Parameters
|
1077
|
+
----------
|
1078
|
+
packages : Dict[str, str], default: {}
|
1079
|
+
Packages to use for this step. The key is the name of the package
|
1080
|
+
and the value is the version to use.
|
1081
|
+
python : str, optional, default: None
|
1082
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1083
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1084
|
+
"""
|
1085
|
+
...
|
1086
|
+
|
1087
|
+
@typing.overload
|
1088
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1089
|
+
...
|
1090
|
+
|
1091
|
+
@typing.overload
|
1092
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1093
|
+
...
|
1094
|
+
|
1095
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1096
|
+
"""
|
1097
|
+
Specifies the PyPI packages for the step.
|
1098
|
+
|
1099
|
+
Information in this decorator will augment any
|
1100
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1101
|
+
you can use `@pypi_base` to set packages required by all
|
1102
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1300
1103
|
|
1301
1104
|
Parameters
|
1302
1105
|
----------
|
1303
|
-
|
1304
|
-
|
1305
|
-
|
1306
|
-
|
1307
|
-
|
1308
|
-
|
1106
|
+
packages : Dict[str, str], default: {}
|
1107
|
+
Packages to use for this step. The key is the name of the package
|
1108
|
+
and the value is the version to use.
|
1109
|
+
python : str, optional, default: None
|
1110
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1111
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1309
1112
|
"""
|
1310
1113
|
...
|
1311
1114
|
|
1312
1115
|
@typing.overload
|
1313
|
-
def
|
1116
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1117
|
+
"""
|
1118
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1119
|
+
|
1120
|
+
Parameters
|
1121
|
+
----------
|
1122
|
+
vars : Dict[str, str], default {}
|
1123
|
+
Dictionary of environment variables to set.
|
1124
|
+
"""
|
1314
1125
|
...
|
1315
1126
|
|
1316
1127
|
@typing.overload
|
1317
|
-
def
|
1128
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1318
1129
|
...
|
1319
1130
|
|
1320
|
-
|
1131
|
+
@typing.overload
|
1132
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1133
|
+
...
|
1134
|
+
|
1135
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1321
1136
|
"""
|
1322
|
-
Specifies
|
1137
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1323
1138
|
|
1324
|
-
|
1139
|
+
Parameters
|
1140
|
+
----------
|
1141
|
+
vars : Dict[str, str], default {}
|
1142
|
+
Dictionary of environment variables to set.
|
1143
|
+
"""
|
1144
|
+
...
|
1145
|
+
|
1146
|
+
@typing.overload
|
1147
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1148
|
+
"""
|
1149
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1325
1150
|
|
1326
|
-
|
1327
|
-
|
1328
|
-
|
1151
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1152
|
+
|
1153
|
+
Parameters
|
1154
|
+
----------
|
1155
|
+
type : str, default 'default'
|
1156
|
+
Card type.
|
1157
|
+
id : str, optional, default None
|
1158
|
+
If multiple cards are present, use this id to identify this card.
|
1159
|
+
options : Dict[str, Any], default {}
|
1160
|
+
Options passed to the card. The contents depend on the card type.
|
1161
|
+
timeout : int, default 45
|
1162
|
+
Interrupt reporting if it takes more than this many seconds.
|
1329
1163
|
|
1330
|
-
|
1331
|
-
|
1164
|
+
|
1165
|
+
"""
|
1166
|
+
...
|
1167
|
+
|
1168
|
+
@typing.overload
|
1169
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1170
|
+
...
|
1171
|
+
|
1172
|
+
@typing.overload
|
1173
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1174
|
+
...
|
1175
|
+
|
1176
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
1177
|
+
"""
|
1178
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1179
|
+
|
1180
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1332
1181
|
|
1333
1182
|
Parameters
|
1334
1183
|
----------
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
|
1184
|
+
type : str, default 'default'
|
1185
|
+
Card type.
|
1186
|
+
id : str, optional, default None
|
1187
|
+
If multiple cards are present, use this id to identify this card.
|
1188
|
+
options : Dict[str, Any], default {}
|
1189
|
+
Options passed to the card. The contents depend on the card type.
|
1190
|
+
timeout : int, default 45
|
1191
|
+
Interrupt reporting if it takes more than this many seconds.
|
1192
|
+
|
1193
|
+
|
1341
1194
|
"""
|
1342
1195
|
...
|
1343
1196
|
|
1344
1197
|
@typing.overload
|
1345
|
-
def
|
1198
|
+
def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1346
1199
|
"""
|
1347
|
-
Specifies
|
1200
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
1348
1201
|
|
1349
|
-
|
1350
|
-
|
1351
|
-
|
1352
|
-
|
1202
|
+
Parameters
|
1203
|
+
----------
|
1204
|
+
cpu : int, default 1
|
1205
|
+
Number of CPUs required for this step. If `@resources` is
|
1206
|
+
also present, the maximum value from all decorators is used.
|
1207
|
+
gpu : int, default 0
|
1208
|
+
Number of GPUs required for this step. If `@resources` is
|
1209
|
+
also present, the maximum value from all decorators is used.
|
1210
|
+
memory : int, default 4096
|
1211
|
+
Memory size (in MB) required for this step. If
|
1212
|
+
`@resources` is also present, the maximum value from all decorators is
|
1213
|
+
used.
|
1214
|
+
image : str, optional, default None
|
1215
|
+
Docker image to use when launching on AWS Batch. If not specified, and
|
1216
|
+
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
1217
|
+
not, a default Docker image mapping to the current version of Python is used.
|
1218
|
+
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
1219
|
+
AWS Batch Job Queue to submit the job to.
|
1220
|
+
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
1221
|
+
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
1222
|
+
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
1223
|
+
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
1224
|
+
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
1225
|
+
shared_memory : int, optional, default None
|
1226
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1227
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1228
|
+
max_swap : int, optional, default None
|
1229
|
+
The total amount of swap memory (in MiB) a container can use for this
|
1230
|
+
step. This parameter is translated to the `--memory-swap` option in
|
1231
|
+
Docker where the value is the sum of the container memory plus the
|
1232
|
+
`max_swap` value.
|
1233
|
+
swappiness : int, optional, default None
|
1234
|
+
This allows you to tune memory swappiness behavior for this step.
|
1235
|
+
A swappiness value of 0 causes swapping not to happen unless absolutely
|
1236
|
+
necessary. A swappiness value of 100 causes pages to be swapped very
|
1237
|
+
aggressively. Accepted values are whole numbers between 0 and 100.
|
1238
|
+
use_tmpfs : bool, default False
|
1239
|
+
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
1240
|
+
not available on Fargate compute environments
|
1241
|
+
tmpfs_tempdir : bool, default True
|
1242
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
1243
|
+
tmpfs_size : int, optional, default None
|
1244
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
1245
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
1246
|
+
memory allocated for this step.
|
1247
|
+
tmpfs_path : str, optional, default None
|
1248
|
+
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
1249
|
+
inferentia : int, default 0
|
1250
|
+
Number of Inferentia chips required for this step.
|
1251
|
+
trainium : int, default None
|
1252
|
+
Alias for inferentia. Use only one of the two.
|
1253
|
+
efa : int, default 0
|
1254
|
+
Number of elastic fabric adapter network devices to attach to container
|
1255
|
+
ephemeral_storage : int, default None
|
1256
|
+
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
1257
|
+
This is only relevant for Fargate compute environments
|
1258
|
+
log_driver: str, optional, default None
|
1259
|
+
The log driver to use for the Amazon ECS container.
|
1260
|
+
log_options: List[str], optional, default None
|
1261
|
+
List of strings containing options for the chosen log driver. The configurable values
|
1262
|
+
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
1263
|
+
Example: [`awslogs-group:aws/batch/job`]
|
1264
|
+
"""
|
1265
|
+
...
|
1266
|
+
|
1267
|
+
@typing.overload
|
1268
|
+
def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1269
|
+
...
|
1270
|
+
|
1271
|
+
@typing.overload
|
1272
|
+
def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1273
|
+
...
|
1274
|
+
|
1275
|
+
def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
|
1276
|
+
"""
|
1277
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
1353
1278
|
|
1354
1279
|
Parameters
|
1355
1280
|
----------
|
1356
|
-
|
1357
|
-
|
1358
|
-
|
1359
|
-
|
1360
|
-
|
1361
|
-
|
1281
|
+
cpu : int, default 1
|
1282
|
+
Number of CPUs required for this step. If `@resources` is
|
1283
|
+
also present, the maximum value from all decorators is used.
|
1284
|
+
gpu : int, default 0
|
1285
|
+
Number of GPUs required for this step. If `@resources` is
|
1286
|
+
also present, the maximum value from all decorators is used.
|
1287
|
+
memory : int, default 4096
|
1288
|
+
Memory size (in MB) required for this step. If
|
1289
|
+
`@resources` is also present, the maximum value from all decorators is
|
1290
|
+
used.
|
1291
|
+
image : str, optional, default None
|
1292
|
+
Docker image to use when launching on AWS Batch. If not specified, and
|
1293
|
+
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
1294
|
+
not, a default Docker image mapping to the current version of Python is used.
|
1295
|
+
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
1296
|
+
AWS Batch Job Queue to submit the job to.
|
1297
|
+
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
1298
|
+
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
1299
|
+
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
1300
|
+
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
1301
|
+
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
1302
|
+
shared_memory : int, optional, default None
|
1303
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1304
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1305
|
+
max_swap : int, optional, default None
|
1306
|
+
The total amount of swap memory (in MiB) a container can use for this
|
1307
|
+
step. This parameter is translated to the `--memory-swap` option in
|
1308
|
+
Docker where the value is the sum of the container memory plus the
|
1309
|
+
`max_swap` value.
|
1310
|
+
swappiness : int, optional, default None
|
1311
|
+
This allows you to tune memory swappiness behavior for this step.
|
1312
|
+
A swappiness value of 0 causes swapping not to happen unless absolutely
|
1313
|
+
necessary. A swappiness value of 100 causes pages to be swapped very
|
1314
|
+
aggressively. Accepted values are whole numbers between 0 and 100.
|
1315
|
+
use_tmpfs : bool, default False
|
1316
|
+
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
1317
|
+
not available on Fargate compute environments
|
1318
|
+
tmpfs_tempdir : bool, default True
|
1319
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
1320
|
+
tmpfs_size : int, optional, default None
|
1321
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
1322
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
1323
|
+
memory allocated for this step.
|
1324
|
+
tmpfs_path : str, optional, default None
|
1325
|
+
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
1326
|
+
inferentia : int, default 0
|
1327
|
+
Number of Inferentia chips required for this step.
|
1328
|
+
trainium : int, default None
|
1329
|
+
Alias for inferentia. Use only one of the two.
|
1330
|
+
efa : int, default 0
|
1331
|
+
Number of elastic fabric adapter network devices to attach to container
|
1332
|
+
ephemeral_storage : int, default None
|
1333
|
+
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
1334
|
+
This is only relevant for Fargate compute environments
|
1335
|
+
log_driver: str, optional, default None
|
1336
|
+
The log driver to use for the Amazon ECS container.
|
1337
|
+
log_options: List[str], optional, default None
|
1338
|
+
List of strings containing options for the chosen log driver. The configurable values
|
1339
|
+
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
1340
|
+
Example: [`awslogs-group:aws/batch/job`]
|
1362
1341
|
"""
|
1363
1342
|
...
|
1364
1343
|
|
1365
1344
|
@typing.overload
|
1366
|
-
def
|
1345
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1346
|
+
"""
|
1347
|
+
Specifies that the step will success under all circumstances.
|
1348
|
+
|
1349
|
+
The decorator will create an optional artifact, specified by `var`, which
|
1350
|
+
contains the exception raised. You can use it to detect the presence
|
1351
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
1352
|
+
are missing.
|
1353
|
+
|
1354
|
+
Parameters
|
1355
|
+
----------
|
1356
|
+
var : str, optional, default None
|
1357
|
+
Name of the artifact in which to store the caught exception.
|
1358
|
+
If not specified, the exception is not stored.
|
1359
|
+
print_exception : bool, default True
|
1360
|
+
Determines whether or not the exception is printed to
|
1361
|
+
stdout when caught.
|
1362
|
+
"""
|
1367
1363
|
...
|
1368
1364
|
|
1369
1365
|
@typing.overload
|
1370
|
-
def
|
1366
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1371
1367
|
...
|
1372
1368
|
|
1373
|
-
|
1369
|
+
@typing.overload
|
1370
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1371
|
+
...
|
1372
|
+
|
1373
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
1374
1374
|
"""
|
1375
|
-
Specifies the
|
1375
|
+
Specifies that the step will success under all circumstances.
|
1376
1376
|
|
1377
|
-
|
1378
|
-
|
1379
|
-
|
1380
|
-
|
1377
|
+
The decorator will create an optional artifact, specified by `var`, which
|
1378
|
+
contains the exception raised. You can use it to detect the presence
|
1379
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
1380
|
+
are missing.
|
1381
1381
|
|
1382
1382
|
Parameters
|
1383
1383
|
----------
|
1384
|
-
|
1385
|
-
|
1386
|
-
|
1387
|
-
|
1388
|
-
|
1389
|
-
|
1384
|
+
var : str, optional, default None
|
1385
|
+
Name of the artifact in which to store the caught exception.
|
1386
|
+
If not specified, the exception is not stored.
|
1387
|
+
print_exception : bool, default True
|
1388
|
+
Determines whether or not the exception is printed to
|
1389
|
+
stdout when caught.
|
1390
|
+
"""
|
1391
|
+
...
|
1392
|
+
|
1393
|
+
@typing.overload
|
1394
|
+
def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1395
|
+
"""
|
1396
|
+
Specifies the flow(s) that this flow depends on.
|
1397
|
+
|
1398
|
+
```
|
1399
|
+
@trigger_on_finish(flow='FooFlow')
|
1400
|
+
```
|
1401
|
+
or
|
1402
|
+
```
|
1403
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1404
|
+
```
|
1405
|
+
This decorator respects the @project decorator and triggers the flow
|
1406
|
+
when upstream runs within the same namespace complete successfully
|
1407
|
+
|
1408
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1409
|
+
by specifying the fully qualified project_flow_name.
|
1410
|
+
```
|
1411
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1412
|
+
```
|
1413
|
+
or
|
1414
|
+
```
|
1415
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1416
|
+
```
|
1417
|
+
|
1418
|
+
You can also specify just the project or project branch (other values will be
|
1419
|
+
inferred from the current project or project branch):
|
1420
|
+
```
|
1421
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1422
|
+
```
|
1423
|
+
|
1424
|
+
Note that `branch` is typically one of:
|
1425
|
+
- `prod`
|
1426
|
+
- `user.bob`
|
1427
|
+
- `test.my_experiment`
|
1428
|
+
- `prod.staging`
|
1429
|
+
|
1430
|
+
Parameters
|
1431
|
+
----------
|
1432
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1433
|
+
Upstream flow dependency for this flow.
|
1434
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1435
|
+
Upstream flow dependencies for this flow.
|
1436
|
+
options : Dict[str, Any], default {}
|
1437
|
+
Backend-specific configuration for tuning eventing behavior.
|
1438
|
+
|
1439
|
+
|
1440
|
+
"""
|
1441
|
+
...
|
1442
|
+
|
1443
|
+
@typing.overload
|
1444
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1445
|
+
...
|
1446
|
+
|
1447
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1448
|
+
"""
|
1449
|
+
Specifies the flow(s) that this flow depends on.
|
1450
|
+
|
1451
|
+
```
|
1452
|
+
@trigger_on_finish(flow='FooFlow')
|
1453
|
+
```
|
1454
|
+
or
|
1455
|
+
```
|
1456
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1457
|
+
```
|
1458
|
+
This decorator respects the @project decorator and triggers the flow
|
1459
|
+
when upstream runs within the same namespace complete successfully
|
1460
|
+
|
1461
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1462
|
+
by specifying the fully qualified project_flow_name.
|
1463
|
+
```
|
1464
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1465
|
+
```
|
1466
|
+
or
|
1467
|
+
```
|
1468
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1469
|
+
```
|
1470
|
+
|
1471
|
+
You can also specify just the project or project branch (other values will be
|
1472
|
+
inferred from the current project or project branch):
|
1473
|
+
```
|
1474
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1475
|
+
```
|
1476
|
+
|
1477
|
+
Note that `branch` is typically one of:
|
1478
|
+
- `prod`
|
1479
|
+
- `user.bob`
|
1480
|
+
- `test.my_experiment`
|
1481
|
+
- `prod.staging`
|
1482
|
+
|
1483
|
+
Parameters
|
1484
|
+
----------
|
1485
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1486
|
+
Upstream flow dependency for this flow.
|
1487
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1488
|
+
Upstream flow dependencies for this flow.
|
1489
|
+
options : Dict[str, Any], default {}
|
1490
|
+
Backend-specific configuration for tuning eventing behavior.
|
1491
|
+
|
1492
|
+
|
1493
|
+
"""
|
1494
|
+
...
|
1495
|
+
|
1496
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1497
|
+
"""
|
1498
|
+
Specifies what flows belong to the same project.
|
1499
|
+
|
1500
|
+
A project-specific namespace is created for all flows that
|
1501
|
+
use the same `@project(name)`.
|
1502
|
+
|
1503
|
+
Parameters
|
1504
|
+
----------
|
1505
|
+
name : str
|
1506
|
+
Project name. Make sure that the name is unique amongst all
|
1507
|
+
projects that use the same production scheduler. The name may
|
1508
|
+
contain only lowercase alphanumeric characters and underscores.
|
1509
|
+
|
1510
|
+
|
1390
1511
|
"""
|
1391
1512
|
...
|
1392
1513
|
|
@@ -1521,93 +1642,62 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1521
1642
|
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1522
1643
|
```
|
1523
1644
|
|
1524
|
-
Parameters
|
1525
|
-
----------
|
1526
|
-
event : Union[str, Dict[str, Any]], optional, default None
|
1527
|
-
Event dependency for this flow.
|
1528
|
-
events : List[Union[str, Dict[str, Any]]], default []
|
1529
|
-
Events dependency for this flow.
|
1530
|
-
options : Dict[str, Any], default {}
|
1531
|
-
Backend-specific configuration for tuning eventing behavior.
|
1532
|
-
|
1533
|
-
|
1534
|
-
"""
|
1535
|
-
...
|
1536
|
-
|
1537
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1538
|
-
"""
|
1539
|
-
Specifies what flows belong to the same project.
|
1540
|
-
|
1541
|
-
A project-specific namespace is created for all flows that
|
1542
|
-
use the same `@project(name)`.
|
1543
|
-
|
1544
|
-
Parameters
|
1545
|
-
----------
|
1546
|
-
name : str
|
1547
|
-
Project name. Make sure that the name is unique amongst all
|
1548
|
-
projects that use the same production scheduler. The name may
|
1549
|
-
contain only lowercase alphanumeric characters and underscores.
|
1550
|
-
|
1645
|
+
Parameters
|
1646
|
+
----------
|
1647
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1648
|
+
Event dependency for this flow.
|
1649
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1650
|
+
Events dependency for this flow.
|
1651
|
+
options : Dict[str, Any], default {}
|
1652
|
+
Backend-specific configuration for tuning eventing behavior.
|
1653
|
+
|
1551
1654
|
|
1552
1655
|
"""
|
1553
1656
|
...
|
1554
1657
|
|
1555
1658
|
@typing.overload
|
1556
|
-
def
|
1659
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1557
1660
|
"""
|
1558
|
-
Specifies the
|
1559
|
-
|
1560
|
-
Use `@conda_base` to set common libraries required by all
|
1561
|
-
steps and use `@conda` to specify step-specific additions.
|
1661
|
+
Specifies the PyPI packages for all steps of the flow.
|
1562
1662
|
|
1663
|
+
Use `@pypi_base` to set common packages required by all
|
1664
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1563
1665
|
Parameters
|
1564
1666
|
----------
|
1565
|
-
packages : Dict[str, str], default {}
|
1667
|
+
packages : Dict[str, str], default: {}
|
1566
1668
|
Packages to use for this flow. The key is the name of the package
|
1567
1669
|
and the value is the version to use.
|
1568
|
-
|
1569
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1570
|
-
python : str, optional, default None
|
1670
|
+
python : str, optional, default: None
|
1571
1671
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1572
1672
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1573
|
-
disabled : bool, default False
|
1574
|
-
If set to True, disables Conda.
|
1575
1673
|
"""
|
1576
1674
|
...
|
1577
1675
|
|
1578
1676
|
@typing.overload
|
1579
|
-
def
|
1677
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1580
1678
|
...
|
1581
1679
|
|
1582
|
-
def
|
1680
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1583
1681
|
"""
|
1584
|
-
Specifies the
|
1585
|
-
|
1586
|
-
Use `@conda_base` to set common libraries required by all
|
1587
|
-
steps and use `@conda` to specify step-specific additions.
|
1682
|
+
Specifies the PyPI packages for all steps of the flow.
|
1588
1683
|
|
1684
|
+
Use `@pypi_base` to set common packages required by all
|
1685
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1589
1686
|
Parameters
|
1590
1687
|
----------
|
1591
|
-
packages : Dict[str, str], default {}
|
1688
|
+
packages : Dict[str, str], default: {}
|
1592
1689
|
Packages to use for this flow. The key is the name of the package
|
1593
1690
|
and the value is the version to use.
|
1594
|
-
|
1595
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1596
|
-
python : str, optional, default None
|
1691
|
+
python : str, optional, default: None
|
1597
1692
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1598
1693
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1599
|
-
disabled : bool, default False
|
1600
|
-
If set to True, disables Conda.
|
1601
1694
|
"""
|
1602
1695
|
...
|
1603
1696
|
|
1604
|
-
def
|
1697
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1605
1698
|
"""
|
1606
|
-
The `@
|
1607
|
-
|
1608
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1609
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1610
|
-
starts only after all sensors finish.
|
1699
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1700
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1611
1701
|
|
1612
1702
|
Parameters
|
1613
1703
|
----------
|
@@ -1628,25 +1718,31 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1628
1718
|
Name of the sensor on Airflow
|
1629
1719
|
description : str
|
1630
1720
|
Description of sensor in the Airflow UI
|
1631
|
-
|
1632
|
-
The
|
1633
|
-
|
1634
|
-
|
1635
|
-
|
1636
|
-
|
1637
|
-
|
1638
|
-
|
1639
|
-
|
1640
|
-
|
1641
|
-
|
1642
|
-
|
1721
|
+
external_dag_id : str
|
1722
|
+
The dag_id that contains the task you want to wait for.
|
1723
|
+
external_task_ids : List[str]
|
1724
|
+
The list of task_ids that you want to wait for.
|
1725
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1726
|
+
allowed_states : List[str]
|
1727
|
+
Iterable of allowed states, (Default: ['success'])
|
1728
|
+
failed_states : List[str]
|
1729
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1730
|
+
execution_delta : datetime.timedelta
|
1731
|
+
time difference with the previous execution to look at,
|
1732
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1733
|
+
check_existence: bool
|
1734
|
+
Set to True to check if the external task exists or check if
|
1735
|
+
the DAG to wait for exists. (Default: True)
|
1643
1736
|
"""
|
1644
1737
|
...
|
1645
1738
|
|
1646
|
-
def
|
1739
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1647
1740
|
"""
|
1648
|
-
The `@
|
1649
|
-
This decorator only works when a flow is scheduled on Airflow
|
1741
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1742
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1743
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1744
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1745
|
+
starts only after all sensors finish.
|
1650
1746
|
|
1651
1747
|
Parameters
|
1652
1748
|
----------
|
@@ -1667,163 +1763,67 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1667
1763
|
Name of the sensor on Airflow
|
1668
1764
|
description : str
|
1669
1765
|
Description of sensor in the Airflow UI
|
1670
|
-
|
1671
|
-
The
|
1672
|
-
|
1673
|
-
|
1674
|
-
|
1675
|
-
|
1676
|
-
|
1677
|
-
|
1678
|
-
|
1679
|
-
|
1680
|
-
|
1681
|
-
|
1682
|
-
check_existence: bool
|
1683
|
-
Set to True to check if the external task exists or check if
|
1684
|
-
the DAG to wait for exists. (Default: True)
|
1766
|
+
bucket_key : Union[str, List[str]]
|
1767
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1768
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1769
|
+
bucket_name : str
|
1770
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1771
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1772
|
+
wildcard_match : bool
|
1773
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1774
|
+
aws_conn_id : str
|
1775
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1776
|
+
verify : bool
|
1777
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1685
1778
|
"""
|
1686
1779
|
...
|
1687
1780
|
|
1688
1781
|
@typing.overload
|
1689
|
-
def
|
1782
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1690
1783
|
"""
|
1691
|
-
Specifies the
|
1784
|
+
Specifies the Conda environment for all steps of the flow.
|
1785
|
+
|
1786
|
+
Use `@conda_base` to set common libraries required by all
|
1787
|
+
steps and use `@conda` to specify step-specific additions.
|
1692
1788
|
|
1693
|
-
Use `@pypi_base` to set common packages required by all
|
1694
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1695
1789
|
Parameters
|
1696
1790
|
----------
|
1697
|
-
packages : Dict[str, str], default
|
1791
|
+
packages : Dict[str, str], default {}
|
1698
1792
|
Packages to use for this flow. The key is the name of the package
|
1699
1793
|
and the value is the version to use.
|
1700
|
-
|
1794
|
+
libraries : Dict[str, str], default {}
|
1795
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1796
|
+
python : str, optional, default None
|
1701
1797
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1702
1798
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1799
|
+
disabled : bool, default False
|
1800
|
+
If set to True, disables Conda.
|
1703
1801
|
"""
|
1704
1802
|
...
|
1705
1803
|
|
1706
1804
|
@typing.overload
|
1707
|
-
def
|
1805
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1708
1806
|
...
|
1709
1807
|
|
1710
|
-
def
|
1808
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1711
1809
|
"""
|
1712
|
-
Specifies the
|
1810
|
+
Specifies the Conda environment for all steps of the flow.
|
1811
|
+
|
1812
|
+
Use `@conda_base` to set common libraries required by all
|
1813
|
+
steps and use `@conda` to specify step-specific additions.
|
1713
1814
|
|
1714
|
-
Use `@pypi_base` to set common packages required by all
|
1715
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1716
1815
|
Parameters
|
1717
1816
|
----------
|
1718
|
-
packages : Dict[str, str], default
|
1817
|
+
packages : Dict[str, str], default {}
|
1719
1818
|
Packages to use for this flow. The key is the name of the package
|
1720
1819
|
and the value is the version to use.
|
1721
|
-
|
1820
|
+
libraries : Dict[str, str], default {}
|
1821
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1822
|
+
python : str, optional, default None
|
1722
1823
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1723
1824
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1724
|
-
|
1725
|
-
|
1726
|
-
|
1727
|
-
@typing.overload
|
1728
|
-
def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1729
|
-
"""
|
1730
|
-
Specifies the flow(s) that this flow depends on.
|
1731
|
-
|
1732
|
-
```
|
1733
|
-
@trigger_on_finish(flow='FooFlow')
|
1734
|
-
```
|
1735
|
-
or
|
1736
|
-
```
|
1737
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1738
|
-
```
|
1739
|
-
This decorator respects the @project decorator and triggers the flow
|
1740
|
-
when upstream runs within the same namespace complete successfully
|
1741
|
-
|
1742
|
-
Additionally, you can specify project aware upstream flow dependencies
|
1743
|
-
by specifying the fully qualified project_flow_name.
|
1744
|
-
```
|
1745
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1746
|
-
```
|
1747
|
-
or
|
1748
|
-
```
|
1749
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1750
|
-
```
|
1751
|
-
|
1752
|
-
You can also specify just the project or project branch (other values will be
|
1753
|
-
inferred from the current project or project branch):
|
1754
|
-
```
|
1755
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1756
|
-
```
|
1757
|
-
|
1758
|
-
Note that `branch` is typically one of:
|
1759
|
-
- `prod`
|
1760
|
-
- `user.bob`
|
1761
|
-
- `test.my_experiment`
|
1762
|
-
- `prod.staging`
|
1763
|
-
|
1764
|
-
Parameters
|
1765
|
-
----------
|
1766
|
-
flow : Union[str, Dict[str, str]], optional, default None
|
1767
|
-
Upstream flow dependency for this flow.
|
1768
|
-
flows : List[Union[str, Dict[str, str]]], default []
|
1769
|
-
Upstream flow dependencies for this flow.
|
1770
|
-
options : Dict[str, Any], default {}
|
1771
|
-
Backend-specific configuration for tuning eventing behavior.
|
1772
|
-
|
1773
|
-
|
1774
|
-
"""
|
1775
|
-
...
|
1776
|
-
|
1777
|
-
@typing.overload
|
1778
|
-
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1779
|
-
...
|
1780
|
-
|
1781
|
-
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1782
|
-
"""
|
1783
|
-
Specifies the flow(s) that this flow depends on.
|
1784
|
-
|
1785
|
-
```
|
1786
|
-
@trigger_on_finish(flow='FooFlow')
|
1787
|
-
```
|
1788
|
-
or
|
1789
|
-
```
|
1790
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1791
|
-
```
|
1792
|
-
This decorator respects the @project decorator and triggers the flow
|
1793
|
-
when upstream runs within the same namespace complete successfully
|
1794
|
-
|
1795
|
-
Additionally, you can specify project aware upstream flow dependencies
|
1796
|
-
by specifying the fully qualified project_flow_name.
|
1797
|
-
```
|
1798
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1799
|
-
```
|
1800
|
-
or
|
1801
|
-
```
|
1802
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1803
|
-
```
|
1804
|
-
|
1805
|
-
You can also specify just the project or project branch (other values will be
|
1806
|
-
inferred from the current project or project branch):
|
1807
|
-
```
|
1808
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1809
|
-
```
|
1810
|
-
|
1811
|
-
Note that `branch` is typically one of:
|
1812
|
-
- `prod`
|
1813
|
-
- `user.bob`
|
1814
|
-
- `test.my_experiment`
|
1815
|
-
- `prod.staging`
|
1816
|
-
|
1817
|
-
Parameters
|
1818
|
-
----------
|
1819
|
-
flow : Union[str, Dict[str, str]], optional, default None
|
1820
|
-
Upstream flow dependency for this flow.
|
1821
|
-
flows : List[Union[str, Dict[str, str]]], default []
|
1822
|
-
Upstream flow dependencies for this flow.
|
1823
|
-
options : Dict[str, Any], default {}
|
1824
|
-
Backend-specific configuration for tuning eventing behavior.
|
1825
|
-
|
1826
|
-
|
1825
|
+
disabled : bool, default False
|
1826
|
+
If set to True, disables Conda.
|
1827
1827
|
"""
|
1828
1828
|
...
|
1829
1829
|
|