metaflow-stubs 2.12.13__py2.py3-none-any.whl → 2.12.15__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +358 -358
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +4 -4
- metaflow-stubs/client/core.pyi +7 -7
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +3 -3
- metaflow-stubs/metadata/metadata.pyi +3 -3
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +23 -23
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +3 -3
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +8 -6
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +7 -7
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +20 -5
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +3 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +19 -4
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +6 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_job.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +6 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +5 -5
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +21 -6
- metaflow-stubs/runner/metaflow_runner.pyi +5 -5
- metaflow-stubs/runner/nbdeploy.pyi +4 -4
- metaflow-stubs/runner/nbrun.pyi +4 -4
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +4 -4
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +3 -3
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- metaflow-stubs/version.pyi +2 -2
- {metaflow_stubs-2.12.13.dist-info → metaflow_stubs-2.12.15.dist-info}/METADATA +2 -2
- metaflow_stubs-2.12.15.dist-info/RECORD +150 -0
- metaflow_stubs-2.12.13.dist-info/RECORD +0 -150
- {metaflow_stubs-2.12.13.dist-info → metaflow_stubs-2.12.15.dist-info}/WHEEL +0 -0
- {metaflow_stubs-2.12.13.dist-info → metaflow_stubs-2.12.15.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,25 +1,25 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-08-
|
3
|
+
# MF version: 2.12.15 #
|
4
|
+
# Generated on 2024-08-22T20:18:55.258441 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import
|
12
|
-
import metaflow.client.core
|
11
|
+
import metaflow.runner.metaflow_runner
|
13
12
|
import metaflow.parameters
|
14
13
|
import io
|
15
|
-
import metaflow.metaflow_current
|
16
14
|
import metaflow.flowspec
|
17
|
-
import
|
15
|
+
import metaflow.client.core
|
18
16
|
import metaflow.events
|
17
|
+
import datetime
|
19
18
|
import metaflow.datastore.inputs
|
19
|
+
import typing
|
20
20
|
import metaflow.plugins.datatools.s3.s3
|
21
|
-
import metaflow.runner.metaflow_runner
|
22
21
|
import metaflow._vendor.click.types
|
22
|
+
import metaflow.metaflow_current
|
23
23
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
24
24
|
StepFlag = typing.NewType("StepFlag", bool)
|
25
25
|
|
@@ -727,112 +727,6 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
727
727
|
"""
|
728
728
|
...
|
729
729
|
|
730
|
-
@typing.overload
|
731
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
732
|
-
"""
|
733
|
-
Specifies the PyPI packages for the step.
|
734
|
-
|
735
|
-
Information in this decorator will augment any
|
736
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
737
|
-
you can use `@pypi_base` to set packages required by all
|
738
|
-
steps and use `@pypi` to specify step-specific overrides.
|
739
|
-
|
740
|
-
Parameters
|
741
|
-
----------
|
742
|
-
packages : Dict[str, str], default: {}
|
743
|
-
Packages to use for this step. The key is the name of the package
|
744
|
-
and the value is the version to use.
|
745
|
-
python : str, optional, default: None
|
746
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
747
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
748
|
-
"""
|
749
|
-
...
|
750
|
-
|
751
|
-
@typing.overload
|
752
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
753
|
-
...
|
754
|
-
|
755
|
-
@typing.overload
|
756
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
757
|
-
...
|
758
|
-
|
759
|
-
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
760
|
-
"""
|
761
|
-
Specifies the PyPI packages for the step.
|
762
|
-
|
763
|
-
Information in this decorator will augment any
|
764
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
765
|
-
you can use `@pypi_base` to set packages required by all
|
766
|
-
steps and use `@pypi` to specify step-specific overrides.
|
767
|
-
|
768
|
-
Parameters
|
769
|
-
----------
|
770
|
-
packages : Dict[str, str], default: {}
|
771
|
-
Packages to use for this step. The key is the name of the package
|
772
|
-
and the value is the version to use.
|
773
|
-
python : str, optional, default: None
|
774
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
775
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
776
|
-
"""
|
777
|
-
...
|
778
|
-
|
779
|
-
@typing.overload
|
780
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
781
|
-
"""
|
782
|
-
Specifies the Conda environment for the step.
|
783
|
-
|
784
|
-
Information in this decorator will augment any
|
785
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
786
|
-
you can use `@conda_base` to set packages required by all
|
787
|
-
steps and use `@conda` to specify step-specific overrides.
|
788
|
-
|
789
|
-
Parameters
|
790
|
-
----------
|
791
|
-
packages : Dict[str, str], default {}
|
792
|
-
Packages to use for this step. The key is the name of the package
|
793
|
-
and the value is the version to use.
|
794
|
-
libraries : Dict[str, str], default {}
|
795
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
796
|
-
python : str, optional, default None
|
797
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
798
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
799
|
-
disabled : bool, default False
|
800
|
-
If set to True, disables @conda.
|
801
|
-
"""
|
802
|
-
...
|
803
|
-
|
804
|
-
@typing.overload
|
805
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
806
|
-
...
|
807
|
-
|
808
|
-
@typing.overload
|
809
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
810
|
-
...
|
811
|
-
|
812
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
813
|
-
"""
|
814
|
-
Specifies the Conda environment for the step.
|
815
|
-
|
816
|
-
Information in this decorator will augment any
|
817
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
818
|
-
you can use `@conda_base` to set packages required by all
|
819
|
-
steps and use `@conda` to specify step-specific overrides.
|
820
|
-
|
821
|
-
Parameters
|
822
|
-
----------
|
823
|
-
packages : Dict[str, str], default {}
|
824
|
-
Packages to use for this step. The key is the name of the package
|
825
|
-
and the value is the version to use.
|
826
|
-
libraries : Dict[str, str], default {}
|
827
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
828
|
-
python : str, optional, default None
|
829
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
830
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
831
|
-
disabled : bool, default False
|
832
|
-
If set to True, disables @conda.
|
833
|
-
"""
|
834
|
-
...
|
835
|
-
|
836
730
|
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
837
731
|
"""
|
838
732
|
Specifies that this step should execute on Kubernetes.
|
@@ -949,59 +843,51 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
949
843
|
...
|
950
844
|
|
951
845
|
@typing.overload
|
952
|
-
def
|
846
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
953
847
|
"""
|
954
|
-
Specifies
|
955
|
-
|
956
|
-
This decorator is useful if this step may hang indefinitely.
|
957
|
-
|
958
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
959
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
960
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
848
|
+
Specifies that the step will success under all circumstances.
|
961
849
|
|
962
|
-
|
963
|
-
|
850
|
+
The decorator will create an optional artifact, specified by `var`, which
|
851
|
+
contains the exception raised. You can use it to detect the presence
|
852
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
853
|
+
are missing.
|
964
854
|
|
965
855
|
Parameters
|
966
856
|
----------
|
967
|
-
|
968
|
-
|
969
|
-
|
970
|
-
|
971
|
-
|
972
|
-
|
857
|
+
var : str, optional, default None
|
858
|
+
Name of the artifact in which to store the caught exception.
|
859
|
+
If not specified, the exception is not stored.
|
860
|
+
print_exception : bool, default True
|
861
|
+
Determines whether or not the exception is printed to
|
862
|
+
stdout when caught.
|
973
863
|
"""
|
974
864
|
...
|
975
865
|
|
976
866
|
@typing.overload
|
977
|
-
def
|
867
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
978
868
|
...
|
979
869
|
|
980
870
|
@typing.overload
|
981
|
-
def
|
871
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
982
872
|
...
|
983
873
|
|
984
|
-
def
|
874
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
985
875
|
"""
|
986
|
-
Specifies
|
987
|
-
|
988
|
-
This decorator is useful if this step may hang indefinitely.
|
989
|
-
|
990
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
991
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
992
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
876
|
+
Specifies that the step will success under all circumstances.
|
993
877
|
|
994
|
-
|
995
|
-
|
878
|
+
The decorator will create an optional artifact, specified by `var`, which
|
879
|
+
contains the exception raised. You can use it to detect the presence
|
880
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
881
|
+
are missing.
|
996
882
|
|
997
883
|
Parameters
|
998
884
|
----------
|
999
|
-
|
1000
|
-
|
1001
|
-
|
1002
|
-
|
1003
|
-
|
1004
|
-
|
885
|
+
var : str, optional, default None
|
886
|
+
Name of the artifact in which to store the caught exception.
|
887
|
+
If not specified, the exception is not stored.
|
888
|
+
print_exception : bool, default True
|
889
|
+
Determines whether or not the exception is printed to
|
890
|
+
stdout when caught.
|
1005
891
|
"""
|
1006
892
|
...
|
1007
893
|
|
@@ -1083,73 +969,118 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
1083
969
|
...
|
1084
970
|
|
1085
971
|
@typing.overload
|
1086
|
-
def
|
972
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1087
973
|
"""
|
1088
|
-
Specifies
|
1089
|
-
the execution of a step.
|
974
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1090
975
|
|
1091
976
|
Parameters
|
1092
977
|
----------
|
1093
|
-
|
1094
|
-
|
978
|
+
vars : Dict[str, str], default {}
|
979
|
+
Dictionary of environment variables to set.
|
1095
980
|
"""
|
1096
981
|
...
|
1097
982
|
|
1098
983
|
@typing.overload
|
1099
|
-
def
|
984
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1100
985
|
...
|
1101
986
|
|
1102
987
|
@typing.overload
|
1103
|
-
def
|
988
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1104
989
|
...
|
1105
990
|
|
1106
|
-
def
|
991
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1107
992
|
"""
|
1108
|
-
Specifies
|
1109
|
-
the execution of a step.
|
993
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1110
994
|
|
1111
995
|
Parameters
|
1112
996
|
----------
|
1113
|
-
|
1114
|
-
|
997
|
+
vars : Dict[str, str], default {}
|
998
|
+
Dictionary of environment variables to set.
|
1115
999
|
"""
|
1116
1000
|
...
|
1117
1001
|
|
1118
1002
|
@typing.overload
|
1119
|
-
def
|
1003
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1120
1004
|
"""
|
1121
|
-
|
1005
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1006
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1007
|
+
"""
|
1008
|
+
...
|
1009
|
+
|
1010
|
+
@typing.overload
|
1011
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1012
|
+
...
|
1013
|
+
|
1014
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1015
|
+
"""
|
1016
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1017
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1018
|
+
"""
|
1019
|
+
...
|
1020
|
+
|
1021
|
+
@typing.overload
|
1022
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1023
|
+
"""
|
1024
|
+
Specifies a timeout for your step.
|
1122
1025
|
|
1123
|
-
|
1026
|
+
This decorator is useful if this step may hang indefinitely.
|
1124
1027
|
|
1125
|
-
|
1126
|
-
|
1127
|
-
|
1128
|
-
Card type.
|
1129
|
-
id : str, optional, default None
|
1130
|
-
If multiple cards are present, use this id to identify this card.
|
1131
|
-
options : Dict[str, Any], default {}
|
1132
|
-
Options passed to the card. The contents depend on the card type.
|
1133
|
-
timeout : int, default 45
|
1134
|
-
Interrupt reporting if it takes more than this many seconds.
|
1028
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1029
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1030
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1135
1031
|
|
1032
|
+
Note that all the values specified in parameters are added together so if you specify
|
1033
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1136
1034
|
|
1035
|
+
Parameters
|
1036
|
+
----------
|
1037
|
+
seconds : int, default 0
|
1038
|
+
Number of seconds to wait prior to timing out.
|
1039
|
+
minutes : int, default 0
|
1040
|
+
Number of minutes to wait prior to timing out.
|
1041
|
+
hours : int, default 0
|
1042
|
+
Number of hours to wait prior to timing out.
|
1137
1043
|
"""
|
1138
1044
|
...
|
1139
1045
|
|
1140
1046
|
@typing.overload
|
1141
|
-
def
|
1047
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1142
1048
|
...
|
1143
1049
|
|
1144
1050
|
@typing.overload
|
1145
|
-
def
|
1051
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1146
1052
|
...
|
1147
1053
|
|
1148
|
-
def
|
1054
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1149
1055
|
"""
|
1150
|
-
|
1056
|
+
Specifies a timeout for your step.
|
1151
1057
|
|
1152
|
-
|
1058
|
+
This decorator is useful if this step may hang indefinitely.
|
1059
|
+
|
1060
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1061
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1062
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1063
|
+
|
1064
|
+
Note that all the values specified in parameters are added together so if you specify
|
1065
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1066
|
+
|
1067
|
+
Parameters
|
1068
|
+
----------
|
1069
|
+
seconds : int, default 0
|
1070
|
+
Number of seconds to wait prior to timing out.
|
1071
|
+
minutes : int, default 0
|
1072
|
+
Number of minutes to wait prior to timing out.
|
1073
|
+
hours : int, default 0
|
1074
|
+
Number of hours to wait prior to timing out.
|
1075
|
+
"""
|
1076
|
+
...
|
1077
|
+
|
1078
|
+
@typing.overload
|
1079
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1080
|
+
"""
|
1081
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1082
|
+
|
1083
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1153
1084
|
|
1154
1085
|
Parameters
|
1155
1086
|
----------
|
@@ -1167,33 +1098,31 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
1167
1098
|
...
|
1168
1099
|
|
1169
1100
|
@typing.overload
|
1170
|
-
def
|
1171
|
-
"""
|
1172
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1173
|
-
|
1174
|
-
Parameters
|
1175
|
-
----------
|
1176
|
-
vars : Dict[str, str], default {}
|
1177
|
-
Dictionary of environment variables to set.
|
1178
|
-
"""
|
1179
|
-
...
|
1180
|
-
|
1181
|
-
@typing.overload
|
1182
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1101
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1183
1102
|
...
|
1184
1103
|
|
1185
1104
|
@typing.overload
|
1186
|
-
def
|
1105
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1187
1106
|
...
|
1188
1107
|
|
1189
|
-
def
|
1108
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
1190
1109
|
"""
|
1191
|
-
|
1110
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
1111
|
+
|
1112
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
1192
1113
|
|
1193
1114
|
Parameters
|
1194
1115
|
----------
|
1195
|
-
|
1196
|
-
|
1116
|
+
type : str, default 'default'
|
1117
|
+
Card type.
|
1118
|
+
id : str, optional, default None
|
1119
|
+
If multiple cards are present, use this id to identify this card.
|
1120
|
+
options : Dict[str, Any], default {}
|
1121
|
+
Options passed to the card. The contents depend on the card type.
|
1122
|
+
timeout : int, default 45
|
1123
|
+
Interrupt reporting if it takes more than this many seconds.
|
1124
|
+
|
1125
|
+
|
1197
1126
|
"""
|
1198
1127
|
...
|
1199
1128
|
|
@@ -1345,109 +1274,190 @@ def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1345
1274
|
...
|
1346
1275
|
|
1347
1276
|
@typing.overload
|
1348
|
-
def
|
1277
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1349
1278
|
"""
|
1350
|
-
|
1351
|
-
|
1279
|
+
Specifies the PyPI packages for the step.
|
1280
|
+
|
1281
|
+
Information in this decorator will augment any
|
1282
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1283
|
+
you can use `@pypi_base` to set packages required by all
|
1284
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1285
|
+
|
1286
|
+
Parameters
|
1287
|
+
----------
|
1288
|
+
packages : Dict[str, str], default: {}
|
1289
|
+
Packages to use for this step. The key is the name of the package
|
1290
|
+
and the value is the version to use.
|
1291
|
+
python : str, optional, default: None
|
1292
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1293
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1352
1294
|
"""
|
1353
1295
|
...
|
1354
1296
|
|
1355
1297
|
@typing.overload
|
1356
|
-
def
|
1298
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1357
1299
|
...
|
1358
1300
|
|
1359
|
-
|
1301
|
+
@typing.overload
|
1302
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1303
|
+
...
|
1304
|
+
|
1305
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1360
1306
|
"""
|
1361
|
-
|
1362
|
-
|
1307
|
+
Specifies the PyPI packages for the step.
|
1308
|
+
|
1309
|
+
Information in this decorator will augment any
|
1310
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1311
|
+
you can use `@pypi_base` to set packages required by all
|
1312
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1313
|
+
|
1314
|
+
Parameters
|
1315
|
+
----------
|
1316
|
+
packages : Dict[str, str], default: {}
|
1317
|
+
Packages to use for this step. The key is the name of the package
|
1318
|
+
and the value is the version to use.
|
1319
|
+
python : str, optional, default: None
|
1320
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1321
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1363
1322
|
"""
|
1364
1323
|
...
|
1365
1324
|
|
1366
1325
|
@typing.overload
|
1367
|
-
def
|
1326
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1368
1327
|
"""
|
1369
|
-
Specifies
|
1370
|
-
|
1371
|
-
The decorator will create an optional artifact, specified by `var`, which
|
1372
|
-
contains the exception raised. You can use it to detect the presence
|
1373
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
1374
|
-
are missing.
|
1328
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1329
|
+
the execution of a step.
|
1375
1330
|
|
1376
1331
|
Parameters
|
1377
1332
|
----------
|
1378
|
-
|
1379
|
-
|
1380
|
-
If not specified, the exception is not stored.
|
1381
|
-
print_exception : bool, default True
|
1382
|
-
Determines whether or not the exception is printed to
|
1383
|
-
stdout when caught.
|
1333
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1334
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1384
1335
|
"""
|
1385
1336
|
...
|
1386
1337
|
|
1387
1338
|
@typing.overload
|
1388
|
-
def
|
1339
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1389
1340
|
...
|
1390
1341
|
|
1391
1342
|
@typing.overload
|
1392
|
-
def
|
1343
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1393
1344
|
...
|
1394
1345
|
|
1395
|
-
def
|
1346
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
1396
1347
|
"""
|
1397
|
-
Specifies
|
1398
|
-
|
1399
|
-
The decorator will create an optional artifact, specified by `var`, which
|
1400
|
-
contains the exception raised. You can use it to detect the presence
|
1401
|
-
of errors, indicating that all happy-path artifacts produced by the step
|
1402
|
-
are missing.
|
1348
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
1349
|
+
the execution of a step.
|
1403
1350
|
|
1404
1351
|
Parameters
|
1405
1352
|
----------
|
1406
|
-
|
1407
|
-
|
1408
|
-
If not specified, the exception is not stored.
|
1409
|
-
print_exception : bool, default True
|
1410
|
-
Determines whether or not the exception is printed to
|
1411
|
-
stdout when caught.
|
1353
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
1354
|
+
List of secret specs, defining how the secrets are to be retrieved
|
1412
1355
|
"""
|
1413
1356
|
...
|
1414
1357
|
|
1415
1358
|
@typing.overload
|
1416
|
-
def
|
1359
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1417
1360
|
"""
|
1418
|
-
Specifies the
|
1361
|
+
Specifies the Conda environment for the step.
|
1362
|
+
|
1363
|
+
Information in this decorator will augment any
|
1364
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1365
|
+
you can use `@conda_base` to set packages required by all
|
1366
|
+
steps and use `@conda` to specify step-specific overrides.
|
1419
1367
|
|
1420
|
-
Use `@pypi_base` to set common packages required by all
|
1421
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1422
1368
|
Parameters
|
1423
1369
|
----------
|
1424
|
-
packages : Dict[str, str], default
|
1425
|
-
Packages to use for this
|
1370
|
+
packages : Dict[str, str], default {}
|
1371
|
+
Packages to use for this step. The key is the name of the package
|
1426
1372
|
and the value is the version to use.
|
1427
|
-
|
1373
|
+
libraries : Dict[str, str], default {}
|
1374
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1375
|
+
python : str, optional, default None
|
1428
1376
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1429
1377
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1378
|
+
disabled : bool, default False
|
1379
|
+
If set to True, disables @conda.
|
1430
1380
|
"""
|
1431
1381
|
...
|
1432
1382
|
|
1433
1383
|
@typing.overload
|
1434
|
-
def
|
1384
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1435
1385
|
...
|
1436
1386
|
|
1437
|
-
|
1387
|
+
@typing.overload
|
1388
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1389
|
+
...
|
1390
|
+
|
1391
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1438
1392
|
"""
|
1439
|
-
Specifies the
|
1393
|
+
Specifies the Conda environment for the step.
|
1394
|
+
|
1395
|
+
Information in this decorator will augment any
|
1396
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1397
|
+
you can use `@conda_base` to set packages required by all
|
1398
|
+
steps and use `@conda` to specify step-specific overrides.
|
1440
1399
|
|
1441
|
-
Use `@pypi_base` to set common packages required by all
|
1442
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1443
1400
|
Parameters
|
1444
1401
|
----------
|
1445
|
-
packages : Dict[str, str], default
|
1446
|
-
Packages to use for this
|
1402
|
+
packages : Dict[str, str], default {}
|
1403
|
+
Packages to use for this step. The key is the name of the package
|
1447
1404
|
and the value is the version to use.
|
1448
|
-
|
1405
|
+
libraries : Dict[str, str], default {}
|
1406
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1407
|
+
python : str, optional, default None
|
1449
1408
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1450
1409
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1410
|
+
disabled : bool, default False
|
1411
|
+
If set to True, disables @conda.
|
1412
|
+
"""
|
1413
|
+
...
|
1414
|
+
|
1415
|
+
@typing.overload
|
1416
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1417
|
+
"""
|
1418
|
+
Specifies the times when the flow should be run when running on a
|
1419
|
+
production scheduler.
|
1420
|
+
|
1421
|
+
Parameters
|
1422
|
+
----------
|
1423
|
+
hourly : bool, default False
|
1424
|
+
Run the workflow hourly.
|
1425
|
+
daily : bool, default True
|
1426
|
+
Run the workflow daily.
|
1427
|
+
weekly : bool, default False
|
1428
|
+
Run the workflow weekly.
|
1429
|
+
cron : str, optional, default None
|
1430
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1431
|
+
specified by this expression.
|
1432
|
+
timezone : str, optional, default None
|
1433
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1434
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1435
|
+
"""
|
1436
|
+
...
|
1437
|
+
|
1438
|
+
@typing.overload
|
1439
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1440
|
+
...
|
1441
|
+
|
1442
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1443
|
+
"""
|
1444
|
+
Specifies the times when the flow should be run when running on a
|
1445
|
+
production scheduler.
|
1446
|
+
|
1447
|
+
Parameters
|
1448
|
+
----------
|
1449
|
+
hourly : bool, default False
|
1450
|
+
Run the workflow hourly.
|
1451
|
+
daily : bool, default True
|
1452
|
+
Run the workflow daily.
|
1453
|
+
weekly : bool, default False
|
1454
|
+
Run the workflow weekly.
|
1455
|
+
cron : str, optional, default None
|
1456
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1457
|
+
specified by this expression.
|
1458
|
+
timezone : str, optional, default None
|
1459
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1460
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1451
1461
|
"""
|
1452
1462
|
...
|
1453
1463
|
|
@@ -1554,115 +1564,6 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1554
1564
|
"""
|
1555
1565
|
...
|
1556
1566
|
|
1557
|
-
@typing.overload
|
1558
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1559
|
-
"""
|
1560
|
-
Specifies the times when the flow should be run when running on a
|
1561
|
-
production scheduler.
|
1562
|
-
|
1563
|
-
Parameters
|
1564
|
-
----------
|
1565
|
-
hourly : bool, default False
|
1566
|
-
Run the workflow hourly.
|
1567
|
-
daily : bool, default True
|
1568
|
-
Run the workflow daily.
|
1569
|
-
weekly : bool, default False
|
1570
|
-
Run the workflow weekly.
|
1571
|
-
cron : str, optional, default None
|
1572
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1573
|
-
specified by this expression.
|
1574
|
-
timezone : str, optional, default None
|
1575
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1576
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1577
|
-
"""
|
1578
|
-
...
|
1579
|
-
|
1580
|
-
@typing.overload
|
1581
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1582
|
-
...
|
1583
|
-
|
1584
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1585
|
-
"""
|
1586
|
-
Specifies the times when the flow should be run when running on a
|
1587
|
-
production scheduler.
|
1588
|
-
|
1589
|
-
Parameters
|
1590
|
-
----------
|
1591
|
-
hourly : bool, default False
|
1592
|
-
Run the workflow hourly.
|
1593
|
-
daily : bool, default True
|
1594
|
-
Run the workflow daily.
|
1595
|
-
weekly : bool, default False
|
1596
|
-
Run the workflow weekly.
|
1597
|
-
cron : str, optional, default None
|
1598
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1599
|
-
specified by this expression.
|
1600
|
-
timezone : str, optional, default None
|
1601
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1602
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1603
|
-
"""
|
1604
|
-
...
|
1605
|
-
|
1606
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1607
|
-
"""
|
1608
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1609
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1610
|
-
|
1611
|
-
Parameters
|
1612
|
-
----------
|
1613
|
-
timeout : int
|
1614
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1615
|
-
poke_interval : int
|
1616
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1617
|
-
mode : str
|
1618
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1619
|
-
exponential_backoff : bool
|
1620
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1621
|
-
pool : str
|
1622
|
-
the slot pool this task should run in,
|
1623
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1624
|
-
soft_fail : bool
|
1625
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1626
|
-
name : str
|
1627
|
-
Name of the sensor on Airflow
|
1628
|
-
description : str
|
1629
|
-
Description of sensor in the Airflow UI
|
1630
|
-
external_dag_id : str
|
1631
|
-
The dag_id that contains the task you want to wait for.
|
1632
|
-
external_task_ids : List[str]
|
1633
|
-
The list of task_ids that you want to wait for.
|
1634
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
1635
|
-
allowed_states : List[str]
|
1636
|
-
Iterable of allowed states, (Default: ['success'])
|
1637
|
-
failed_states : List[str]
|
1638
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
1639
|
-
execution_delta : datetime.timedelta
|
1640
|
-
time difference with the previous execution to look at,
|
1641
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
1642
|
-
check_existence: bool
|
1643
|
-
Set to True to check if the external task exists or check if
|
1644
|
-
the DAG to wait for exists. (Default: True)
|
1645
|
-
"""
|
1646
|
-
...
|
1647
|
-
|
1648
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1649
|
-
"""
|
1650
|
-
Specifies what flows belong to the same project.
|
1651
|
-
|
1652
|
-
A project-specific namespace is created for all flows that
|
1653
|
-
use the same `@project(name)`.
|
1654
|
-
|
1655
|
-
Parameters
|
1656
|
-
----------
|
1657
|
-
name : str
|
1658
|
-
Project name. Make sure that the name is unique amongst all
|
1659
|
-
projects that use the same production scheduler. The name may
|
1660
|
-
contain only lowercase alphanumeric characters and underscores.
|
1661
|
-
|
1662
|
-
|
1663
|
-
"""
|
1664
|
-
...
|
1665
|
-
|
1666
1567
|
@typing.overload
|
1667
1568
|
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1668
1569
|
"""
|
@@ -1807,6 +1708,66 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1807
1708
|
"""
|
1808
1709
|
...
|
1809
1710
|
|
1711
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1712
|
+
"""
|
1713
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1714
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1715
|
+
|
1716
|
+
Parameters
|
1717
|
+
----------
|
1718
|
+
timeout : int
|
1719
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1720
|
+
poke_interval : int
|
1721
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1722
|
+
mode : str
|
1723
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1724
|
+
exponential_backoff : bool
|
1725
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1726
|
+
pool : str
|
1727
|
+
the slot pool this task should run in,
|
1728
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1729
|
+
soft_fail : bool
|
1730
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1731
|
+
name : str
|
1732
|
+
Name of the sensor on Airflow
|
1733
|
+
description : str
|
1734
|
+
Description of sensor in the Airflow UI
|
1735
|
+
external_dag_id : str
|
1736
|
+
The dag_id that contains the task you want to wait for.
|
1737
|
+
external_task_ids : List[str]
|
1738
|
+
The list of task_ids that you want to wait for.
|
1739
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1740
|
+
allowed_states : List[str]
|
1741
|
+
Iterable of allowed states, (Default: ['success'])
|
1742
|
+
failed_states : List[str]
|
1743
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1744
|
+
execution_delta : datetime.timedelta
|
1745
|
+
time difference with the previous execution to look at,
|
1746
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1747
|
+
check_existence: bool
|
1748
|
+
Set to True to check if the external task exists or check if
|
1749
|
+
the DAG to wait for exists. (Default: True)
|
1750
|
+
"""
|
1751
|
+
...
|
1752
|
+
|
1753
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1754
|
+
"""
|
1755
|
+
Specifies what flows belong to the same project.
|
1756
|
+
|
1757
|
+
A project-specific namespace is created for all flows that
|
1758
|
+
use the same `@project(name)`.
|
1759
|
+
|
1760
|
+
Parameters
|
1761
|
+
----------
|
1762
|
+
name : str
|
1763
|
+
Project name. Make sure that the name is unique amongst all
|
1764
|
+
projects that use the same production scheduler. The name may
|
1765
|
+
contain only lowercase alphanumeric characters and underscores.
|
1766
|
+
|
1767
|
+
|
1768
|
+
"""
|
1769
|
+
...
|
1770
|
+
|
1810
1771
|
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1811
1772
|
"""
|
1812
1773
|
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
@@ -1849,6 +1810,45 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1849
1810
|
"""
|
1850
1811
|
...
|
1851
1812
|
|
1813
|
+
@typing.overload
|
1814
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1815
|
+
"""
|
1816
|
+
Specifies the PyPI packages for all steps of the flow.
|
1817
|
+
|
1818
|
+
Use `@pypi_base` to set common packages required by all
|
1819
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1820
|
+
Parameters
|
1821
|
+
----------
|
1822
|
+
packages : Dict[str, str], default: {}
|
1823
|
+
Packages to use for this flow. The key is the name of the package
|
1824
|
+
and the value is the version to use.
|
1825
|
+
python : str, optional, default: None
|
1826
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1827
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1828
|
+
"""
|
1829
|
+
...
|
1830
|
+
|
1831
|
+
@typing.overload
|
1832
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1833
|
+
...
|
1834
|
+
|
1835
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1836
|
+
"""
|
1837
|
+
Specifies the PyPI packages for all steps of the flow.
|
1838
|
+
|
1839
|
+
Use `@pypi_base` to set common packages required by all
|
1840
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1841
|
+
Parameters
|
1842
|
+
----------
|
1843
|
+
packages : Dict[str, str], default: {}
|
1844
|
+
Packages to use for this flow. The key is the name of the package
|
1845
|
+
and the value is the version to use.
|
1846
|
+
python : str, optional, default: None
|
1847
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1848
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1849
|
+
"""
|
1850
|
+
...
|
1851
|
+
|
1852
1852
|
def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
1853
1853
|
"""
|
1854
1854
|
Switch namespace to the one provided.
|
@@ -2800,7 +2800,7 @@ class DataArtifact(metaflow.client.core.MetaflowObject, metaclass=type):
|
|
2800
2800
|
...
|
2801
2801
|
|
2802
2802
|
class Runner(object, metaclass=type):
|
2803
|
-
def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, **kwargs):
|
2803
|
+
def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
|
2804
2804
|
...
|
2805
2805
|
def __enter__(self) -> metaflow.runner.metaflow_runner.Runner:
|
2806
2806
|
...
|
@@ -2892,7 +2892,7 @@ class Runner(object, metaclass=type):
|
|
2892
2892
|
...
|
2893
2893
|
|
2894
2894
|
class NBRunner(object, metaclass=type):
|
2895
|
-
def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", **kwargs):
|
2895
|
+
def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", file_read_timeout: int = 3600, **kwargs):
|
2896
2896
|
...
|
2897
2897
|
def nbrun(self, **kwargs):
|
2898
2898
|
"""
|
@@ -2998,7 +2998,7 @@ class NBRunner(object, metaclass=type):
|
|
2998
2998
|
...
|
2999
2999
|
|
3000
3000
|
class Deployer(object, metaclass=type):
|
3001
|
-
def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, **kwargs):
|
3001
|
+
def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, file_read_timeout: int = 3600, **kwargs):
|
3002
3002
|
...
|
3003
3003
|
def _Deployer__make_function(self, deployer_class):
|
3004
3004
|
"""
|
@@ -3018,7 +3018,7 @@ class Deployer(object, metaclass=type):
|
|
3018
3018
|
...
|
3019
3019
|
|
3020
3020
|
class NBDeployer(object, metaclass=type):
|
3021
|
-
def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", **kwargs):
|
3021
|
+
def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", file_read_timeout: int = 3600, **kwargs):
|
3022
3022
|
...
|
3023
3023
|
def cleanup(self):
|
3024
3024
|
"""
|