ob-metaflow-stubs 3.1__py2.py3-none-any.whl → 3.3__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +425 -425
- metaflow-stubs/cards.pyi +6 -6
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +4 -4
- metaflow-stubs/client/core.pyi +5 -5
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +5 -5
- metaflow-stubs/metadata/metadata.pyi +3 -3
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +10 -2
- metaflow-stubs/metaflow_current.pyi +4 -4
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +3 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +4 -4
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +9 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +3 -3
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- {ob_metaflow_stubs-3.1.dist-info → ob_metaflow_stubs-3.3.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-3.3.dist-info/RECORD +134 -0
- ob_metaflow_stubs-3.1.dist-info/RECORD +0 -134
- {ob_metaflow_stubs-3.1.dist-info → ob_metaflow_stubs-3.3.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-3.1.dist-info → ob_metaflow_stubs-3.3.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,23 +1,23 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.11.
|
4
|
-
# Generated on 2024-05-
|
3
|
+
# MF version: 2.11.14.1+ob(v1) #
|
4
|
+
# Generated on 2024-05-13T17:42:14.215304 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import
|
12
|
-
import metaflow.parameters
|
13
|
-
import metaflow.metaflow_current
|
11
|
+
import typing
|
14
12
|
import datetime
|
15
13
|
import metaflow._vendor.click.types
|
14
|
+
import metaflow.parameters
|
15
|
+
import metaflow.client.core
|
16
|
+
import metaflow.plugins.datatools.s3.s3
|
17
|
+
import metaflow.metaflow_current
|
16
18
|
import metaflow.events
|
17
|
-
import typing
|
18
|
-
import io
|
19
19
|
import metaflow.datastore.inputs
|
20
|
-
import
|
20
|
+
import io
|
21
21
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
22
22
|
StepFlag = typing.NewType("StepFlag", bool)
|
23
23
|
|
@@ -726,59 +726,102 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
726
726
|
...
|
727
727
|
|
728
728
|
@typing.overload
|
729
|
-
def
|
729
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
730
730
|
"""
|
731
|
-
Specifies
|
732
|
-
|
733
|
-
This decorator is useful if this step may hang indefinitely.
|
734
|
-
|
735
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
736
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
737
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
731
|
+
Specifies that the step will success under all circumstances.
|
738
732
|
|
739
|
-
|
740
|
-
|
733
|
+
The decorator will create an optional artifact, specified by `var`, which
|
734
|
+
contains the exception raised. You can use it to detect the presence
|
735
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
736
|
+
are missing.
|
741
737
|
|
742
738
|
Parameters
|
743
739
|
----------
|
744
|
-
|
745
|
-
|
746
|
-
|
747
|
-
|
748
|
-
|
749
|
-
|
740
|
+
var : str, optional, default None
|
741
|
+
Name of the artifact in which to store the caught exception.
|
742
|
+
If not specified, the exception is not stored.
|
743
|
+
print_exception : bool, default True
|
744
|
+
Determines whether or not the exception is printed to
|
745
|
+
stdout when caught.
|
750
746
|
"""
|
751
747
|
...
|
752
748
|
|
753
749
|
@typing.overload
|
754
|
-
def
|
750
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
755
751
|
...
|
756
752
|
|
757
753
|
@typing.overload
|
758
|
-
def
|
754
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
759
755
|
...
|
760
756
|
|
761
|
-
def
|
757
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
762
758
|
"""
|
763
|
-
Specifies
|
759
|
+
Specifies that the step will success under all circumstances.
|
764
760
|
|
765
|
-
|
761
|
+
The decorator will create an optional artifact, specified by `var`, which
|
762
|
+
contains the exception raised. You can use it to detect the presence
|
763
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
764
|
+
are missing.
|
766
765
|
|
767
|
-
|
768
|
-
|
769
|
-
|
766
|
+
Parameters
|
767
|
+
----------
|
768
|
+
var : str, optional, default None
|
769
|
+
Name of the artifact in which to store the caught exception.
|
770
|
+
If not specified, the exception is not stored.
|
771
|
+
print_exception : bool, default True
|
772
|
+
Determines whether or not the exception is printed to
|
773
|
+
stdout when caught.
|
774
|
+
"""
|
775
|
+
...
|
776
|
+
|
777
|
+
@typing.overload
|
778
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
779
|
+
"""
|
780
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
770
781
|
|
771
|
-
Note that
|
772
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
782
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
773
783
|
|
774
784
|
Parameters
|
775
785
|
----------
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
|
780
|
-
|
781
|
-
|
786
|
+
type : str, default 'default'
|
787
|
+
Card type.
|
788
|
+
id : str, optional, default None
|
789
|
+
If multiple cards are present, use this id to identify this card.
|
790
|
+
options : Dict[str, Any], default {}
|
791
|
+
Options passed to the card. The contents depend on the card type.
|
792
|
+
timeout : int, default 45
|
793
|
+
Interrupt reporting if it takes more than this many seconds.
|
794
|
+
|
795
|
+
|
796
|
+
"""
|
797
|
+
...
|
798
|
+
|
799
|
+
@typing.overload
|
800
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
801
|
+
...
|
802
|
+
|
803
|
+
@typing.overload
|
804
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
805
|
+
...
|
806
|
+
|
807
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
808
|
+
"""
|
809
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
810
|
+
|
811
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
812
|
+
|
813
|
+
Parameters
|
814
|
+
----------
|
815
|
+
type : str, default 'default'
|
816
|
+
Card type.
|
817
|
+
id : str, optional, default None
|
818
|
+
If multiple cards are present, use this id to identify this card.
|
819
|
+
options : Dict[str, Any], default {}
|
820
|
+
Options passed to the card. The contents depend on the card type.
|
821
|
+
timeout : int, default 45
|
822
|
+
Interrupt reporting if it takes more than this many seconds.
|
823
|
+
|
824
|
+
|
782
825
|
"""
|
783
826
|
...
|
784
827
|
|
@@ -912,63 +955,6 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
912
955
|
"""
|
913
956
|
...
|
914
957
|
|
915
|
-
@typing.overload
|
916
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
917
|
-
"""
|
918
|
-
Specifies the Conda environment for the step.
|
919
|
-
|
920
|
-
Information in this decorator will augment any
|
921
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
922
|
-
you can use `@conda_base` to set packages required by all
|
923
|
-
steps and use `@conda` to specify step-specific overrides.
|
924
|
-
|
925
|
-
Parameters
|
926
|
-
----------
|
927
|
-
packages : Dict[str, str], default {}
|
928
|
-
Packages to use for this step. The key is the name of the package
|
929
|
-
and the value is the version to use.
|
930
|
-
libraries : Dict[str, str], default {}
|
931
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
932
|
-
python : str, optional, default None
|
933
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
934
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
935
|
-
disabled : bool, default False
|
936
|
-
If set to True, disables @conda.
|
937
|
-
"""
|
938
|
-
...
|
939
|
-
|
940
|
-
@typing.overload
|
941
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
942
|
-
...
|
943
|
-
|
944
|
-
@typing.overload
|
945
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
946
|
-
...
|
947
|
-
|
948
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
949
|
-
"""
|
950
|
-
Specifies the Conda environment for the step.
|
951
|
-
|
952
|
-
Information in this decorator will augment any
|
953
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
954
|
-
you can use `@conda_base` to set packages required by all
|
955
|
-
steps and use `@conda` to specify step-specific overrides.
|
956
|
-
|
957
|
-
Parameters
|
958
|
-
----------
|
959
|
-
packages : Dict[str, str], default {}
|
960
|
-
Packages to use for this step. The key is the name of the package
|
961
|
-
and the value is the version to use.
|
962
|
-
libraries : Dict[str, str], default {}
|
963
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
964
|
-
python : str, optional, default None
|
965
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
966
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
967
|
-
disabled : bool, default False
|
968
|
-
If set to True, disables @conda.
|
969
|
-
"""
|
970
|
-
...
|
971
|
-
|
972
958
|
@typing.overload
|
973
959
|
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
974
960
|
"""
|
@@ -1018,37 +1004,6 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
1018
1004
|
"""
|
1019
1005
|
...
|
1020
1006
|
|
1021
|
-
@typing.overload
|
1022
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1023
|
-
"""
|
1024
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1025
|
-
|
1026
|
-
Parameters
|
1027
|
-
----------
|
1028
|
-
vars : Dict[str, str], default {}
|
1029
|
-
Dictionary of environment variables to set.
|
1030
|
-
"""
|
1031
|
-
...
|
1032
|
-
|
1033
|
-
@typing.overload
|
1034
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1035
|
-
...
|
1036
|
-
|
1037
|
-
@typing.overload
|
1038
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1039
|
-
...
|
1040
|
-
|
1041
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1042
|
-
"""
|
1043
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1044
|
-
|
1045
|
-
Parameters
|
1046
|
-
----------
|
1047
|
-
vars : Dict[str, str], default {}
|
1048
|
-
Dictionary of environment variables to set.
|
1049
|
-
"""
|
1050
|
-
...
|
1051
|
-
|
1052
1007
|
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1053
1008
|
"""
|
1054
1009
|
Specifies that this step should execute on Kubernetes.
|
@@ -1109,57 +1064,235 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
1109
1064
|
...
|
1110
1065
|
|
1111
1066
|
@typing.overload
|
1112
|
-
def
|
1067
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1113
1068
|
"""
|
1114
|
-
Specifies
|
1069
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1115
1070
|
|
1116
1071
|
Parameters
|
1117
1072
|
----------
|
1118
|
-
|
1119
|
-
|
1120
|
-
|
1121
|
-
|
1122
|
-
|
1123
|
-
|
1124
|
-
|
1125
|
-
|
1126
|
-
|
1127
|
-
|
1128
|
-
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1145
|
-
|
1146
|
-
|
1147
|
-
|
1148
|
-
|
1149
|
-
|
1150
|
-
|
1151
|
-
|
1152
|
-
|
1153
|
-
|
1154
|
-
|
1155
|
-
|
1156
|
-
|
1157
|
-
|
1158
|
-
|
1159
|
-
|
1160
|
-
|
1161
|
-
|
1162
|
-
|
1073
|
+
vars : Dict[str, str], default {}
|
1074
|
+
Dictionary of environment variables to set.
|
1075
|
+
"""
|
1076
|
+
...
|
1077
|
+
|
1078
|
+
@typing.overload
|
1079
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1080
|
+
...
|
1081
|
+
|
1082
|
+
@typing.overload
|
1083
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1084
|
+
...
|
1085
|
+
|
1086
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1087
|
+
"""
|
1088
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1089
|
+
|
1090
|
+
Parameters
|
1091
|
+
----------
|
1092
|
+
vars : Dict[str, str], default {}
|
1093
|
+
Dictionary of environment variables to set.
|
1094
|
+
"""
|
1095
|
+
...
|
1096
|
+
|
1097
|
+
@typing.overload
|
1098
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1099
|
+
"""
|
1100
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1101
|
+
the execution of a step.
|
1102
|
+
|
1103
|
+
Parameters
|
1104
|
+
----------
|
1105
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1106
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1107
|
+
"""
|
1108
|
+
...
|
1109
|
+
|
1110
|
+
@typing.overload
|
1111
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1112
|
+
...
|
1113
|
+
|
1114
|
+
@typing.overload
|
1115
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1116
|
+
...
|
1117
|
+
|
1118
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
1119
|
+
"""
|
1120
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1121
|
+
the execution of a step.
|
1122
|
+
|
1123
|
+
Parameters
|
1124
|
+
----------
|
1125
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1126
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1127
|
+
"""
|
1128
|
+
...
|
1129
|
+
|
1130
|
+
@typing.overload
|
1131
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1132
|
+
"""
|
1133
|
+
Specifies the Conda environment for the step.
|
1134
|
+
|
1135
|
+
Information in this decorator will augment any
|
1136
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1137
|
+
you can use `@conda_base` to set packages required by all
|
1138
|
+
steps and use `@conda` to specify step-specific overrides.
|
1139
|
+
|
1140
|
+
Parameters
|
1141
|
+
----------
|
1142
|
+
packages : Dict[str, str], default {}
|
1143
|
+
Packages to use for this step. The key is the name of the package
|
1144
|
+
and the value is the version to use.
|
1145
|
+
libraries : Dict[str, str], default {}
|
1146
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1147
|
+
python : str, optional, default None
|
1148
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1149
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1150
|
+
disabled : bool, default False
|
1151
|
+
If set to True, disables @conda.
|
1152
|
+
"""
|
1153
|
+
...
|
1154
|
+
|
1155
|
+
@typing.overload
|
1156
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1157
|
+
...
|
1158
|
+
|
1159
|
+
@typing.overload
|
1160
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1161
|
+
...
|
1162
|
+
|
1163
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1164
|
+
"""
|
1165
|
+
Specifies the Conda environment for the step.
|
1166
|
+
|
1167
|
+
Information in this decorator will augment any
|
1168
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1169
|
+
you can use `@conda_base` to set packages required by all
|
1170
|
+
steps and use `@conda` to specify step-specific overrides.
|
1171
|
+
|
1172
|
+
Parameters
|
1173
|
+
----------
|
1174
|
+
packages : Dict[str, str], default {}
|
1175
|
+
Packages to use for this step. The key is the name of the package
|
1176
|
+
and the value is the version to use.
|
1177
|
+
libraries : Dict[str, str], default {}
|
1178
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1179
|
+
python : str, optional, default None
|
1180
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1181
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1182
|
+
disabled : bool, default False
|
1183
|
+
If set to True, disables @conda.
|
1184
|
+
"""
|
1185
|
+
...
|
1186
|
+
|
1187
|
+
@typing.overload
|
1188
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1189
|
+
"""
|
1190
|
+
Specifies a timeout for your step.
|
1191
|
+
|
1192
|
+
This decorator is useful if this step may hang indefinitely.
|
1193
|
+
|
1194
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1195
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1196
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1197
|
+
|
1198
|
+
Note that all the values specified in parameters are added together so if you specify
|
1199
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1200
|
+
|
1201
|
+
Parameters
|
1202
|
+
----------
|
1203
|
+
seconds : int, default 0
|
1204
|
+
Number of seconds to wait prior to timing out.
|
1205
|
+
minutes : int, default 0
|
1206
|
+
Number of minutes to wait prior to timing out.
|
1207
|
+
hours : int, default 0
|
1208
|
+
Number of hours to wait prior to timing out.
|
1209
|
+
"""
|
1210
|
+
...
|
1211
|
+
|
1212
|
+
@typing.overload
|
1213
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1214
|
+
...
|
1215
|
+
|
1216
|
+
@typing.overload
|
1217
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1218
|
+
...
|
1219
|
+
|
1220
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1221
|
+
"""
|
1222
|
+
Specifies a timeout for your step.
|
1223
|
+
|
1224
|
+
This decorator is useful if this step may hang indefinitely.
|
1225
|
+
|
1226
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1227
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1228
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1229
|
+
|
1230
|
+
Note that all the values specified in parameters are added together so if you specify
|
1231
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1232
|
+
|
1233
|
+
Parameters
|
1234
|
+
----------
|
1235
|
+
seconds : int, default 0
|
1236
|
+
Number of seconds to wait prior to timing out.
|
1237
|
+
minutes : int, default 0
|
1238
|
+
Number of minutes to wait prior to timing out.
|
1239
|
+
hours : int, default 0
|
1240
|
+
Number of hours to wait prior to timing out.
|
1241
|
+
"""
|
1242
|
+
...
|
1243
|
+
|
1244
|
+
@typing.overload
|
1245
|
+
def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1246
|
+
"""
|
1247
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
1248
|
+
|
1249
|
+
Parameters
|
1250
|
+
----------
|
1251
|
+
cpu : int, default 1
|
1252
|
+
Number of CPUs required for this step. If `@resources` is
|
1253
|
+
also present, the maximum value from all decorators is used.
|
1254
|
+
gpu : int, default 0
|
1255
|
+
Number of GPUs required for this step. If `@resources` is
|
1256
|
+
also present, the maximum value from all decorators is used.
|
1257
|
+
memory : int, default 4096
|
1258
|
+
Memory size (in MB) required for this step. If
|
1259
|
+
`@resources` is also present, the maximum value from all decorators is
|
1260
|
+
used.
|
1261
|
+
image : str, optional, default None
|
1262
|
+
Docker image to use when launching on AWS Batch. If not specified, and
|
1263
|
+
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
1264
|
+
not, a default Docker image mapping to the current version of Python is used.
|
1265
|
+
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
1266
|
+
AWS Batch Job Queue to submit the job to.
|
1267
|
+
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
1268
|
+
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
1269
|
+
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
1270
|
+
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
1271
|
+
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
1272
|
+
shared_memory : int, optional, default None
|
1273
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1274
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1275
|
+
max_swap : int, optional, default None
|
1276
|
+
The total amount of swap memory (in MiB) a container can use for this
|
1277
|
+
step. This parameter is translated to the `--memory-swap` option in
|
1278
|
+
Docker where the value is the sum of the container memory plus the
|
1279
|
+
`max_swap` value.
|
1280
|
+
swappiness : int, optional, default None
|
1281
|
+
This allows you to tune memory swappiness behavior for this step.
|
1282
|
+
A swappiness value of 0 causes swapping not to happen unless absolutely
|
1283
|
+
necessary. A swappiness value of 100 causes pages to be swapped very
|
1284
|
+
aggressively. Accepted values are whole numbers between 0 and 100.
|
1285
|
+
use_tmpfs : bool, default False
|
1286
|
+
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
1287
|
+
not available on Fargate compute environments
|
1288
|
+
tmpfs_tempdir : bool, default True
|
1289
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
1290
|
+
tmpfs_size : int, optional, default None
|
1291
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
1292
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
1293
|
+
memory allocated for this step.
|
1294
|
+
tmpfs_path : str, optional, default None
|
1295
|
+
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
1163
1296
|
inferentia : int, default 0
|
1164
1297
|
Number of Inferentia chips required for this step.
|
1165
1298
|
trainium : int, default None
|
@@ -1251,224 +1384,56 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1251
1384
|
log_options: List[str], optional, default None
|
1252
1385
|
List of strings containing options for the chosen log driver. The configurable values
|
1253
1386
|
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
1254
|
-
Example usage: ["awslogs-group:aws/batch/job"]
|
1255
|
-
"""
|
1256
|
-
...
|
1257
|
-
|
1258
|
-
@typing.overload
|
1259
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1260
|
-
"""
|
1261
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
1262
|
-
the execution of a step.
|
1263
|
-
|
1264
|
-
Parameters
|
1265
|
-
----------
|
1266
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
1267
|
-
List of secret specs, defining how the secrets are to be retrieved
|
1268
|
-
"""
|
1269
|
-
...
|
1270
|
-
|
1271
|
-
@typing.overload
|
1272
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1273
|
-
...
|
1274
|
-
|
1275
|
-
@typing.overload
|
1276
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1277
|
-
...
|
1278
|
-
|
1279
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
1280
|
-
"""
|
1281
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
1282
|
-
the execution of a step.
|
1283
|
-
|
1284
|
-
Parameters
|
1285
|
-
----------
|
1286
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
1287
|
-
List of secret specs, defining how the secrets are to be retrieved
|
1288
|
-
"""
|
1289
|
-
...
|
1290
|
-
|
1291
|
-
@typing.overload
|
1292
|
-
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1293
|
-
"""
|
1294
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1295
|
-
|
1296
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1297
|
-
|
1298
|
-
Parameters
|
1299
|
-
----------
|
1300
|
-
type : str, default 'default'
|
1301
|
-
Card type.
|
1302
|
-
id : str, optional, default None
|
1303
|
-
If multiple cards are present, use this id to identify this card.
|
1304
|
-
options : Dict[str, Any], default {}
|
1305
|
-
Options passed to the card. The contents depend on the card type.
|
1306
|
-
timeout : int, default 45
|
1307
|
-
Interrupt reporting if it takes more than this many seconds.
|
1308
|
-
|
1309
|
-
|
1310
|
-
"""
|
1311
|
-
...
|
1312
|
-
|
1313
|
-
@typing.overload
|
1314
|
-
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1315
|
-
...
|
1316
|
-
|
1317
|
-
@typing.overload
|
1318
|
-
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1319
|
-
...
|
1320
|
-
|
1321
|
-
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
1322
|
-
"""
|
1323
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1324
|
-
|
1325
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1326
|
-
|
1327
|
-
Parameters
|
1328
|
-
----------
|
1329
|
-
type : str, default 'default'
|
1330
|
-
Card type.
|
1331
|
-
id : str, optional, default None
|
1332
|
-
If multiple cards are present, use this id to identify this card.
|
1333
|
-
options : Dict[str, Any], default {}
|
1334
|
-
Options passed to the card. The contents depend on the card type.
|
1335
|
-
timeout : int, default 45
|
1336
|
-
Interrupt reporting if it takes more than this many seconds.
|
1337
|
-
|
1338
|
-
|
1339
|
-
"""
|
1340
|
-
...
|
1341
|
-
|
1342
|
-
@typing.overload
|
1343
|
-
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1344
|
-
"""
|
1345
|
-
Specifies that the step will success under all circumstances.
|
1346
|
-
|
1347
|
-
The decorator will create an optional artifact, specified by `var`, which
|
1348
|
-
contains the exception raised. You can use it to detect the presence
|
1349
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
1350
|
-
are missing.
|
1351
|
-
|
1352
|
-
Parameters
|
1353
|
-
----------
|
1354
|
-
var : str, optional, default None
|
1355
|
-
Name of the artifact in which to store the caught exception.
|
1356
|
-
If not specified, the exception is not stored.
|
1357
|
-
print_exception : bool, default True
|
1358
|
-
Determines whether or not the exception is printed to
|
1359
|
-
stdout when caught.
|
1360
|
-
"""
|
1361
|
-
...
|
1362
|
-
|
1363
|
-
@typing.overload
|
1364
|
-
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1387
|
+
Example usage: ["awslogs-group:aws/batch/job"]
|
1388
|
+
"""
|
1365
1389
|
...
|
1366
1390
|
|
1367
1391
|
@typing.overload
|
1368
|
-
def
|
1369
|
-
...
|
1370
|
-
|
1371
|
-
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
1392
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1372
1393
|
"""
|
1373
|
-
Specifies
|
1394
|
+
Specifies the Conda environment for all steps of the flow.
|
1374
1395
|
|
1375
|
-
|
1376
|
-
|
1377
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
1378
|
-
are missing.
|
1396
|
+
Use `@conda_base` to set common libraries required by all
|
1397
|
+
steps and use `@conda` to specify step-specific additions.
|
1379
1398
|
|
1380
1399
|
Parameters
|
1381
1400
|
----------
|
1382
|
-
|
1383
|
-
|
1384
|
-
|
1385
|
-
|
1386
|
-
|
1387
|
-
|
1401
|
+
packages : Dict[str, str], default {}
|
1402
|
+
Packages to use for this flow. The key is the name of the package
|
1403
|
+
and the value is the version to use.
|
1404
|
+
libraries : Dict[str, str], default {}
|
1405
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1406
|
+
python : str, optional, default None
|
1407
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1408
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1409
|
+
disabled : bool, default False
|
1410
|
+
If set to True, disables Conda.
|
1388
1411
|
"""
|
1389
1412
|
...
|
1390
1413
|
|
1391
|
-
|
1392
|
-
|
1393
|
-
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1394
|
-
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1395
|
-
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1396
|
-
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1397
|
-
starts only after all sensors finish.
|
1398
|
-
|
1399
|
-
Parameters
|
1400
|
-
----------
|
1401
|
-
timeout : int
|
1402
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1403
|
-
poke_interval : int
|
1404
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1405
|
-
mode : str
|
1406
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1407
|
-
exponential_backoff : bool
|
1408
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1409
|
-
pool : str
|
1410
|
-
the slot pool this task should run in,
|
1411
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1412
|
-
soft_fail : bool
|
1413
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1414
|
-
name : str
|
1415
|
-
Name of the sensor on Airflow
|
1416
|
-
description : str
|
1417
|
-
Description of sensor in the Airflow UI
|
1418
|
-
bucket_key : Union[str, List[str]]
|
1419
|
-
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1420
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1421
|
-
bucket_name : str
|
1422
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1423
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1424
|
-
wildcard_match : bool
|
1425
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1426
|
-
aws_conn_id : str
|
1427
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
1428
|
-
verify : bool
|
1429
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1430
|
-
"""
|
1414
|
+
@typing.overload
|
1415
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1431
1416
|
...
|
1432
1417
|
|
1433
|
-
def
|
1418
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1434
1419
|
"""
|
1435
|
-
|
1436
|
-
|
1420
|
+
Specifies the Conda environment for all steps of the flow.
|
1421
|
+
|
1422
|
+
Use `@conda_base` to set common libraries required by all
|
1423
|
+
steps and use `@conda` to specify step-specific additions.
|
1437
1424
|
|
1438
1425
|
Parameters
|
1439
1426
|
----------
|
1440
|
-
|
1441
|
-
|
1442
|
-
|
1443
|
-
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1447
|
-
|
1448
|
-
|
1449
|
-
|
1450
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1451
|
-
soft_fail : bool
|
1452
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1453
|
-
name : str
|
1454
|
-
Name of the sensor on Airflow
|
1455
|
-
description : str
|
1456
|
-
Description of sensor in the Airflow UI
|
1457
|
-
external_dag_id : str
|
1458
|
-
The dag_id that contains the task you want to wait for.
|
1459
|
-
external_task_ids : List[str]
|
1460
|
-
The list of task_ids that you want to wait for.
|
1461
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
1462
|
-
allowed_states : List[str]
|
1463
|
-
Iterable of allowed states, (Default: ['success'])
|
1464
|
-
failed_states : List[str]
|
1465
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
1466
|
-
execution_delta : datetime.timedelta
|
1467
|
-
time difference with the previous execution to look at,
|
1468
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
1469
|
-
check_existence: bool
|
1470
|
-
Set to True to check if the external task exists or check if
|
1471
|
-
the DAG to wait for exists. (Default: True)
|
1427
|
+
packages : Dict[str, str], default {}
|
1428
|
+
Packages to use for this flow. The key is the name of the package
|
1429
|
+
and the value is the version to use.
|
1430
|
+
libraries : Dict[str, str], default {}
|
1431
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1432
|
+
python : str, optional, default None
|
1433
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1434
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1435
|
+
disabled : bool, default False
|
1436
|
+
If set to True, disables Conda.
|
1472
1437
|
"""
|
1473
1438
|
...
|
1474
1439
|
|
@@ -1642,6 +1607,90 @@ def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly:
|
|
1642
1607
|
"""
|
1643
1608
|
...
|
1644
1609
|
|
1610
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1611
|
+
"""
|
1612
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1613
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1614
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1615
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1616
|
+
starts only after all sensors finish.
|
1617
|
+
|
1618
|
+
Parameters
|
1619
|
+
----------
|
1620
|
+
timeout : int
|
1621
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1622
|
+
poke_interval : int
|
1623
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1624
|
+
mode : str
|
1625
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1626
|
+
exponential_backoff : bool
|
1627
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1628
|
+
pool : str
|
1629
|
+
the slot pool this task should run in,
|
1630
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1631
|
+
soft_fail : bool
|
1632
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1633
|
+
name : str
|
1634
|
+
Name of the sensor on Airflow
|
1635
|
+
description : str
|
1636
|
+
Description of sensor in the Airflow UI
|
1637
|
+
bucket_key : Union[str, List[str]]
|
1638
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1639
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1640
|
+
bucket_name : str
|
1641
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1642
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1643
|
+
wildcard_match : bool
|
1644
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1645
|
+
aws_conn_id : str
|
1646
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1647
|
+
verify : bool
|
1648
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1649
|
+
"""
|
1650
|
+
...
|
1651
|
+
|
1652
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1653
|
+
"""
|
1654
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1655
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1656
|
+
|
1657
|
+
Parameters
|
1658
|
+
----------
|
1659
|
+
timeout : int
|
1660
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1661
|
+
poke_interval : int
|
1662
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1663
|
+
mode : str
|
1664
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1665
|
+
exponential_backoff : bool
|
1666
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1667
|
+
pool : str
|
1668
|
+
the slot pool this task should run in,
|
1669
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1670
|
+
soft_fail : bool
|
1671
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1672
|
+
name : str
|
1673
|
+
Name of the sensor on Airflow
|
1674
|
+
description : str
|
1675
|
+
Description of sensor in the Airflow UI
|
1676
|
+
external_dag_id : str
|
1677
|
+
The dag_id that contains the task you want to wait for.
|
1678
|
+
external_task_ids : List[str]
|
1679
|
+
The list of task_ids that you want to wait for.
|
1680
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1681
|
+
allowed_states : List[str]
|
1682
|
+
Iterable of allowed states, (Default: ['success'])
|
1683
|
+
failed_states : List[str]
|
1684
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1685
|
+
execution_delta : datetime.timedelta
|
1686
|
+
time difference with the previous execution to look at,
|
1687
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1688
|
+
check_existence: bool
|
1689
|
+
Set to True to check if the external task exists or check if
|
1690
|
+
the DAG to wait for exists. (Default: True)
|
1691
|
+
"""
|
1692
|
+
...
|
1693
|
+
|
1645
1694
|
@typing.overload
|
1646
1695
|
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1647
1696
|
"""
|
@@ -1776,55 +1825,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1776
1825
|
"""
|
1777
1826
|
...
|
1778
1827
|
|
1779
|
-
@typing.overload
|
1780
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1781
|
-
"""
|
1782
|
-
Specifies the Conda environment for all steps of the flow.
|
1783
|
-
|
1784
|
-
Use `@conda_base` to set common libraries required by all
|
1785
|
-
steps and use `@conda` to specify step-specific additions.
|
1786
|
-
|
1787
|
-
Parameters
|
1788
|
-
----------
|
1789
|
-
packages : Dict[str, str], default {}
|
1790
|
-
Packages to use for this flow. The key is the name of the package
|
1791
|
-
and the value is the version to use.
|
1792
|
-
libraries : Dict[str, str], default {}
|
1793
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1794
|
-
python : str, optional, default None
|
1795
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1796
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1797
|
-
disabled : bool, default False
|
1798
|
-
If set to True, disables Conda.
|
1799
|
-
"""
|
1800
|
-
...
|
1801
|
-
|
1802
|
-
@typing.overload
|
1803
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1804
|
-
...
|
1805
|
-
|
1806
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1807
|
-
"""
|
1808
|
-
Specifies the Conda environment for all steps of the flow.
|
1809
|
-
|
1810
|
-
Use `@conda_base` to set common libraries required by all
|
1811
|
-
steps and use `@conda` to specify step-specific additions.
|
1812
|
-
|
1813
|
-
Parameters
|
1814
|
-
----------
|
1815
|
-
packages : Dict[str, str], default {}
|
1816
|
-
Packages to use for this flow. The key is the name of the package
|
1817
|
-
and the value is the version to use.
|
1818
|
-
libraries : Dict[str, str], default {}
|
1819
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1820
|
-
python : str, optional, default None
|
1821
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1822
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1823
|
-
disabled : bool, default False
|
1824
|
-
If set to True, disables Conda.
|
1825
|
-
"""
|
1826
|
-
...
|
1827
|
-
|
1828
1828
|
def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
1829
1829
|
"""
|
1830
1830
|
Switch namespace to the one provided.
|