ob-metaflow-stubs 5.3__py2.py3-none-any.whl → 5.4__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +428 -428
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +3 -3
- metaflow-stubs/client/__init__.pyi +4 -4
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +4 -4
- metaflow-stubs/metadata/metadata.pyi +2 -2
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +4 -4
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +3 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +4 -4
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +4 -4
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +5 -5
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +6 -6
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +4 -4
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +3 -3
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +4 -4
- metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +4 -4
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +4 -4
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +3 -3
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +4 -4
- metaflow-stubs/runner/metaflow_runner.pyi +5 -5
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +3 -3
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-5.3.dist-info → ob_metaflow_stubs-5.4.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-5.4.dist-info/RECORD +138 -0
- ob_metaflow_stubs-5.3.dist-info/RECORD +0 -138
- {ob_metaflow_stubs-5.3.dist-info → ob_metaflow_stubs-5.4.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-5.3.dist-info → ob_metaflow_stubs-5.4.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,24 +1,24 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-08-
|
3
|
+
# MF version: 2.12.16.1+ob(v1) #
|
4
|
+
# Generated on 2024-08-26T23:03:56.776768 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import
|
11
|
+
import io
|
12
|
+
import metaflow.client.core
|
13
|
+
import metaflow._vendor.click.types
|
12
14
|
import metaflow.parameters
|
13
15
|
import metaflow.plugins.datatools.s3.s3
|
14
|
-
import io
|
15
|
-
import datetime
|
16
16
|
import metaflow.events
|
17
|
-
import
|
17
|
+
import datetime
|
18
18
|
import metaflow.metaflow_current
|
19
|
-
import metaflow.client.core
|
20
19
|
import metaflow.runner.metaflow_runner
|
21
|
-
import metaflow.
|
20
|
+
import metaflow.datastore.inputs
|
21
|
+
import metaflow.flowspec
|
22
22
|
import typing
|
23
23
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
24
24
|
StepFlag = typing.NewType("StepFlag", bool)
|
@@ -727,6 +727,37 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
727
727
|
"""
|
728
728
|
...
|
729
729
|
|
730
|
+
@typing.overload
|
731
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
732
|
+
"""
|
733
|
+
Specifies environment variables to be set prior to the execution of a step.
|
734
|
+
|
735
|
+
Parameters
|
736
|
+
----------
|
737
|
+
vars : Dict[str, str], default {}
|
738
|
+
Dictionary of environment variables to set.
|
739
|
+
"""
|
740
|
+
...
|
741
|
+
|
742
|
+
@typing.overload
|
743
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
744
|
+
...
|
745
|
+
|
746
|
+
@typing.overload
|
747
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
748
|
+
...
|
749
|
+
|
750
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
751
|
+
"""
|
752
|
+
Specifies environment variables to be set prior to the execution of a step.
|
753
|
+
|
754
|
+
Parameters
|
755
|
+
----------
|
756
|
+
vars : Dict[str, str], default {}
|
757
|
+
Dictionary of environment variables to set.
|
758
|
+
"""
|
759
|
+
...
|
760
|
+
|
730
761
|
@typing.overload
|
731
762
|
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
732
763
|
"""
|
@@ -776,6 +807,55 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
776
807
|
"""
|
777
808
|
...
|
778
809
|
|
810
|
+
@typing.overload
|
811
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
812
|
+
"""
|
813
|
+
Specifies the PyPI packages for the step.
|
814
|
+
|
815
|
+
Information in this decorator will augment any
|
816
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
817
|
+
you can use `@pypi_base` to set packages required by all
|
818
|
+
steps and use `@pypi` to specify step-specific overrides.
|
819
|
+
|
820
|
+
Parameters
|
821
|
+
----------
|
822
|
+
packages : Dict[str, str], default: {}
|
823
|
+
Packages to use for this step. The key is the name of the package
|
824
|
+
and the value is the version to use.
|
825
|
+
python : str, optional, default: None
|
826
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
827
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
828
|
+
"""
|
829
|
+
...
|
830
|
+
|
831
|
+
@typing.overload
|
832
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
833
|
+
...
|
834
|
+
|
835
|
+
@typing.overload
|
836
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
837
|
+
...
|
838
|
+
|
839
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
840
|
+
"""
|
841
|
+
Specifies the PyPI packages for the step.
|
842
|
+
|
843
|
+
Information in this decorator will augment any
|
844
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
845
|
+
you can use `@pypi_base` to set packages required by all
|
846
|
+
steps and use `@pypi` to specify step-specific overrides.
|
847
|
+
|
848
|
+
Parameters
|
849
|
+
----------
|
850
|
+
packages : Dict[str, str], default: {}
|
851
|
+
Packages to use for this step. The key is the name of the package
|
852
|
+
and the value is the version to use.
|
853
|
+
python : str, optional, default: None
|
854
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
855
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
856
|
+
"""
|
857
|
+
...
|
858
|
+
|
779
859
|
@typing.overload
|
780
860
|
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
781
861
|
"""
|
@@ -833,6 +913,149 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
833
913
|
"""
|
834
914
|
...
|
835
915
|
|
916
|
+
@typing.overload
|
917
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
918
|
+
"""
|
919
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
920
|
+
the execution of a step.
|
921
|
+
|
922
|
+
Parameters
|
923
|
+
----------
|
924
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
925
|
+
List of secret specs, defining how the secrets are to be retrieved
|
926
|
+
"""
|
927
|
+
...
|
928
|
+
|
929
|
+
@typing.overload
|
930
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
931
|
+
...
|
932
|
+
|
933
|
+
@typing.overload
|
934
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
935
|
+
...
|
936
|
+
|
937
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
938
|
+
"""
|
939
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
940
|
+
the execution of a step.
|
941
|
+
|
942
|
+
Parameters
|
943
|
+
----------
|
944
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
945
|
+
List of secret specs, defining how the secrets are to be retrieved
|
946
|
+
"""
|
947
|
+
...
|
948
|
+
|
949
|
+
@typing.overload
|
950
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
951
|
+
"""
|
952
|
+
Specifies the number of times the task corresponding
|
953
|
+
to a step needs to be retried.
|
954
|
+
|
955
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
956
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
957
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
958
|
+
|
959
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
960
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
961
|
+
ensuring that the flow execution can continue.
|
962
|
+
|
963
|
+
Parameters
|
964
|
+
----------
|
965
|
+
times : int, default 3
|
966
|
+
Number of times to retry this task.
|
967
|
+
minutes_between_retries : int, default 2
|
968
|
+
Number of minutes between retries.
|
969
|
+
"""
|
970
|
+
...
|
971
|
+
|
972
|
+
@typing.overload
|
973
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
974
|
+
...
|
975
|
+
|
976
|
+
@typing.overload
|
977
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
978
|
+
...
|
979
|
+
|
980
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
981
|
+
"""
|
982
|
+
Specifies the number of times the task corresponding
|
983
|
+
to a step needs to be retried.
|
984
|
+
|
985
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
986
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
987
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
988
|
+
|
989
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
990
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
991
|
+
ensuring that the flow execution can continue.
|
992
|
+
|
993
|
+
Parameters
|
994
|
+
----------
|
995
|
+
times : int, default 3
|
996
|
+
Number of times to retry this task.
|
997
|
+
minutes_between_retries : int, default 2
|
998
|
+
Number of minutes between retries.
|
999
|
+
"""
|
1000
|
+
...
|
1001
|
+
|
1002
|
+
@typing.overload
|
1003
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1004
|
+
"""
|
1005
|
+
Specifies the Conda environment for the step.
|
1006
|
+
|
1007
|
+
Information in this decorator will augment any
|
1008
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1009
|
+
you can use `@conda_base` to set packages required by all
|
1010
|
+
steps and use `@conda` to specify step-specific overrides.
|
1011
|
+
|
1012
|
+
Parameters
|
1013
|
+
----------
|
1014
|
+
packages : Dict[str, str], default {}
|
1015
|
+
Packages to use for this step. The key is the name of the package
|
1016
|
+
and the value is the version to use.
|
1017
|
+
libraries : Dict[str, str], default {}
|
1018
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1019
|
+
python : str, optional, default None
|
1020
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1021
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1022
|
+
disabled : bool, default False
|
1023
|
+
If set to True, disables @conda.
|
1024
|
+
"""
|
1025
|
+
...
|
1026
|
+
|
1027
|
+
@typing.overload
|
1028
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1029
|
+
...
|
1030
|
+
|
1031
|
+
@typing.overload
|
1032
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1033
|
+
...
|
1034
|
+
|
1035
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1036
|
+
"""
|
1037
|
+
Specifies the Conda environment for the step.
|
1038
|
+
|
1039
|
+
Information in this decorator will augment any
|
1040
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1041
|
+
you can use `@conda_base` to set packages required by all
|
1042
|
+
steps and use `@conda` to specify step-specific overrides.
|
1043
|
+
|
1044
|
+
Parameters
|
1045
|
+
----------
|
1046
|
+
packages : Dict[str, str], default {}
|
1047
|
+
Packages to use for this step. The key is the name of the package
|
1048
|
+
and the value is the version to use.
|
1049
|
+
libraries : Dict[str, str], default {}
|
1050
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1051
|
+
python : str, optional, default None
|
1052
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1053
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1054
|
+
disabled : bool, default False
|
1055
|
+
If set to True, disables @conda.
|
1056
|
+
"""
|
1057
|
+
...
|
1058
|
+
|
836
1059
|
@typing.overload
|
837
1060
|
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
838
1061
|
"""
|
@@ -910,26 +1133,62 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
910
1133
|
"""
|
911
1134
|
...
|
912
1135
|
|
913
|
-
|
1136
|
+
@typing.overload
|
1137
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
914
1138
|
"""
|
915
|
-
|
916
|
-
|
917
|
-
|
918
|
-
|
919
|
-
|
920
|
-
|
921
|
-
|
922
|
-
|
923
|
-
|
924
|
-
|
925
|
-
|
926
|
-
|
927
|
-
|
928
|
-
|
929
|
-
|
930
|
-
|
931
|
-
|
932
|
-
|
1139
|
+
Internal decorator to support Fast bakery
|
1140
|
+
"""
|
1141
|
+
...
|
1142
|
+
|
1143
|
+
@typing.overload
|
1144
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1145
|
+
...
|
1146
|
+
|
1147
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1148
|
+
"""
|
1149
|
+
Internal decorator to support Fast bakery
|
1150
|
+
"""
|
1151
|
+
...
|
1152
|
+
|
1153
|
+
@typing.overload
|
1154
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1155
|
+
"""
|
1156
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1157
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1158
|
+
"""
|
1159
|
+
...
|
1160
|
+
|
1161
|
+
@typing.overload
|
1162
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1163
|
+
...
|
1164
|
+
|
1165
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1166
|
+
"""
|
1167
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1168
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1169
|
+
"""
|
1170
|
+
...
|
1171
|
+
|
1172
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1173
|
+
"""
|
1174
|
+
Specifies that this step should execute on Kubernetes.
|
1175
|
+
|
1176
|
+
Parameters
|
1177
|
+
----------
|
1178
|
+
cpu : int, default 1
|
1179
|
+
Number of CPUs required for this step. If `@resources` is
|
1180
|
+
also present, the maximum value from all decorators is used.
|
1181
|
+
memory : int, default 4096
|
1182
|
+
Memory size (in MB) required for this step. If
|
1183
|
+
`@resources` is also present, the maximum value from all decorators is
|
1184
|
+
used.
|
1185
|
+
disk : int, default 10240
|
1186
|
+
Disk size (in MB) required for this step. If
|
1187
|
+
`@resources` is also present, the maximum value from all decorators is
|
1188
|
+
used.
|
1189
|
+
image : str, optional, default None
|
1190
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
1191
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
933
1192
|
not, a default Docker image mapping to the current version of Python is used.
|
934
1193
|
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
935
1194
|
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
@@ -972,265 +1231,6 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
972
1231
|
"""
|
973
1232
|
...
|
974
1233
|
|
975
|
-
@typing.overload
|
976
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
977
|
-
"""
|
978
|
-
Decorator prototype for all step decorators. This function gets specialized
|
979
|
-
and imported for all decorators types by _import_plugin_decorators().
|
980
|
-
"""
|
981
|
-
...
|
982
|
-
|
983
|
-
@typing.overload
|
984
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
985
|
-
...
|
986
|
-
|
987
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
988
|
-
"""
|
989
|
-
Decorator prototype for all step decorators. This function gets specialized
|
990
|
-
and imported for all decorators types by _import_plugin_decorators().
|
991
|
-
"""
|
992
|
-
...
|
993
|
-
|
994
|
-
@typing.overload
|
995
|
-
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
996
|
-
"""
|
997
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
998
|
-
the execution of a step.
|
999
|
-
|
1000
|
-
Parameters
|
1001
|
-
----------
|
1002
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
1003
|
-
List of secret specs, defining how the secrets are to be retrieved
|
1004
|
-
"""
|
1005
|
-
...
|
1006
|
-
|
1007
|
-
@typing.overload
|
1008
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1009
|
-
...
|
1010
|
-
|
1011
|
-
@typing.overload
|
1012
|
-
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1013
|
-
...
|
1014
|
-
|
1015
|
-
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
1016
|
-
"""
|
1017
|
-
Specifies secrets to be retrieved and injected as environment variables prior to
|
1018
|
-
the execution of a step.
|
1019
|
-
|
1020
|
-
Parameters
|
1021
|
-
----------
|
1022
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
1023
|
-
List of secret specs, defining how the secrets are to be retrieved
|
1024
|
-
"""
|
1025
|
-
...
|
1026
|
-
|
1027
|
-
@typing.overload
|
1028
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1029
|
-
"""
|
1030
|
-
Specifies the Conda environment for the step.
|
1031
|
-
|
1032
|
-
Information in this decorator will augment any
|
1033
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1034
|
-
you can use `@conda_base` to set packages required by all
|
1035
|
-
steps and use `@conda` to specify step-specific overrides.
|
1036
|
-
|
1037
|
-
Parameters
|
1038
|
-
----------
|
1039
|
-
packages : Dict[str, str], default {}
|
1040
|
-
Packages to use for this step. The key is the name of the package
|
1041
|
-
and the value is the version to use.
|
1042
|
-
libraries : Dict[str, str], default {}
|
1043
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1044
|
-
python : str, optional, default None
|
1045
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1046
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1047
|
-
disabled : bool, default False
|
1048
|
-
If set to True, disables @conda.
|
1049
|
-
"""
|
1050
|
-
...
|
1051
|
-
|
1052
|
-
@typing.overload
|
1053
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1054
|
-
...
|
1055
|
-
|
1056
|
-
@typing.overload
|
1057
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1058
|
-
...
|
1059
|
-
|
1060
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1061
|
-
"""
|
1062
|
-
Specifies the Conda environment for the step.
|
1063
|
-
|
1064
|
-
Information in this decorator will augment any
|
1065
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1066
|
-
you can use `@conda_base` to set packages required by all
|
1067
|
-
steps and use `@conda` to specify step-specific overrides.
|
1068
|
-
|
1069
|
-
Parameters
|
1070
|
-
----------
|
1071
|
-
packages : Dict[str, str], default {}
|
1072
|
-
Packages to use for this step. The key is the name of the package
|
1073
|
-
and the value is the version to use.
|
1074
|
-
libraries : Dict[str, str], default {}
|
1075
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1076
|
-
python : str, optional, default None
|
1077
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1078
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1079
|
-
disabled : bool, default False
|
1080
|
-
If set to True, disables @conda.
|
1081
|
-
"""
|
1082
|
-
...
|
1083
|
-
|
1084
|
-
@typing.overload
|
1085
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1086
|
-
"""
|
1087
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1088
|
-
|
1089
|
-
Parameters
|
1090
|
-
----------
|
1091
|
-
vars : Dict[str, str], default {}
|
1092
|
-
Dictionary of environment variables to set.
|
1093
|
-
"""
|
1094
|
-
...
|
1095
|
-
|
1096
|
-
@typing.overload
|
1097
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1098
|
-
...
|
1099
|
-
|
1100
|
-
@typing.overload
|
1101
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1102
|
-
...
|
1103
|
-
|
1104
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1105
|
-
"""
|
1106
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1107
|
-
|
1108
|
-
Parameters
|
1109
|
-
----------
|
1110
|
-
vars : Dict[str, str], default {}
|
1111
|
-
Dictionary of environment variables to set.
|
1112
|
-
"""
|
1113
|
-
...
|
1114
|
-
|
1115
|
-
@typing.overload
|
1116
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1117
|
-
"""
|
1118
|
-
Specifies the PyPI packages for the step.
|
1119
|
-
|
1120
|
-
Information in this decorator will augment any
|
1121
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1122
|
-
you can use `@pypi_base` to set packages required by all
|
1123
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1124
|
-
|
1125
|
-
Parameters
|
1126
|
-
----------
|
1127
|
-
packages : Dict[str, str], default: {}
|
1128
|
-
Packages to use for this step. The key is the name of the package
|
1129
|
-
and the value is the version to use.
|
1130
|
-
python : str, optional, default: None
|
1131
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1132
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1133
|
-
"""
|
1134
|
-
...
|
1135
|
-
|
1136
|
-
@typing.overload
|
1137
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1138
|
-
...
|
1139
|
-
|
1140
|
-
@typing.overload
|
1141
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1142
|
-
...
|
1143
|
-
|
1144
|
-
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1145
|
-
"""
|
1146
|
-
Specifies the PyPI packages for the step.
|
1147
|
-
|
1148
|
-
Information in this decorator will augment any
|
1149
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1150
|
-
you can use `@pypi_base` to set packages required by all
|
1151
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1152
|
-
|
1153
|
-
Parameters
|
1154
|
-
----------
|
1155
|
-
packages : Dict[str, str], default: {}
|
1156
|
-
Packages to use for this step. The key is the name of the package
|
1157
|
-
and the value is the version to use.
|
1158
|
-
python : str, optional, default: None
|
1159
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1160
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1161
|
-
"""
|
1162
|
-
...
|
1163
|
-
|
1164
|
-
@typing.overload
|
1165
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1166
|
-
"""
|
1167
|
-
Internal decorator to support Fast bakery
|
1168
|
-
"""
|
1169
|
-
...
|
1170
|
-
|
1171
|
-
@typing.overload
|
1172
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1173
|
-
...
|
1174
|
-
|
1175
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1176
|
-
"""
|
1177
|
-
Internal decorator to support Fast bakery
|
1178
|
-
"""
|
1179
|
-
...
|
1180
|
-
|
1181
|
-
@typing.overload
|
1182
|
-
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1183
|
-
"""
|
1184
|
-
Specifies the number of times the task corresponding
|
1185
|
-
to a step needs to be retried.
|
1186
|
-
|
1187
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
1188
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
1189
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
1190
|
-
|
1191
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
1192
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
1193
|
-
ensuring that the flow execution can continue.
|
1194
|
-
|
1195
|
-
Parameters
|
1196
|
-
----------
|
1197
|
-
times : int, default 3
|
1198
|
-
Number of times to retry this task.
|
1199
|
-
minutes_between_retries : int, default 2
|
1200
|
-
Number of minutes between retries.
|
1201
|
-
"""
|
1202
|
-
...
|
1203
|
-
|
1204
|
-
@typing.overload
|
1205
|
-
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1206
|
-
...
|
1207
|
-
|
1208
|
-
@typing.overload
|
1209
|
-
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1210
|
-
...
|
1211
|
-
|
1212
|
-
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
1213
|
-
"""
|
1214
|
-
Specifies the number of times the task corresponding
|
1215
|
-
to a step needs to be retried.
|
1216
|
-
|
1217
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
1218
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
1219
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
1220
|
-
|
1221
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
1222
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
1223
|
-
ensuring that the flow execution can continue.
|
1224
|
-
|
1225
|
-
Parameters
|
1226
|
-
----------
|
1227
|
-
times : int, default 3
|
1228
|
-
Number of times to retry this task.
|
1229
|
-
minutes_between_retries : int, default 2
|
1230
|
-
Number of minutes between retries.
|
1231
|
-
"""
|
1232
|
-
...
|
1233
|
-
|
1234
1234
|
@typing.overload
|
1235
1235
|
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1236
1236
|
"""
|
@@ -1282,6 +1282,45 @@ def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
1282
1282
|
"""
|
1283
1283
|
...
|
1284
1284
|
|
1285
|
+
@typing.overload
|
1286
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1287
|
+
"""
|
1288
|
+
Specifies the PyPI packages for all steps of the flow.
|
1289
|
+
|
1290
|
+
Use `@pypi_base` to set common packages required by all
|
1291
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1292
|
+
Parameters
|
1293
|
+
----------
|
1294
|
+
packages : Dict[str, str], default: {}
|
1295
|
+
Packages to use for this flow. The key is the name of the package
|
1296
|
+
and the value is the version to use.
|
1297
|
+
python : str, optional, default: None
|
1298
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1299
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1300
|
+
"""
|
1301
|
+
...
|
1302
|
+
|
1303
|
+
@typing.overload
|
1304
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1305
|
+
...
|
1306
|
+
|
1307
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1308
|
+
"""
|
1309
|
+
Specifies the PyPI packages for all steps of the flow.
|
1310
|
+
|
1311
|
+
Use `@pypi_base` to set common packages required by all
|
1312
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1313
|
+
Parameters
|
1314
|
+
----------
|
1315
|
+
packages : Dict[str, str], default: {}
|
1316
|
+
Packages to use for this flow. The key is the name of the package
|
1317
|
+
and the value is the version to use.
|
1318
|
+
python : str, optional, default: None
|
1319
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1320
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1321
|
+
"""
|
1322
|
+
...
|
1323
|
+
|
1285
1324
|
@typing.overload
|
1286
1325
|
def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1287
1326
|
"""
|
@@ -1385,37 +1424,6 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1385
1424
|
"""
|
1386
1425
|
...
|
1387
1426
|
|
1388
|
-
def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1389
|
-
"""
|
1390
|
-
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
1391
|
-
|
1392
|
-
User code call
|
1393
|
-
-----------
|
1394
|
-
@nim(
|
1395
|
-
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
1396
|
-
backend='managed'
|
1397
|
-
)
|
1398
|
-
|
1399
|
-
Valid backend options
|
1400
|
-
---------------------
|
1401
|
-
- 'managed': Outerbounds selects a compute provider based on the model.
|
1402
|
-
- 🚧 'dataplane': Run in your account.
|
1403
|
-
|
1404
|
-
Valid model options
|
1405
|
-
----------------
|
1406
|
-
- 'meta/llama3-8b-instruct': 8B parameter model
|
1407
|
-
- 'meta/llama3-70b-instruct': 70B parameter model
|
1408
|
-
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
1409
|
-
|
1410
|
-
Parameters
|
1411
|
-
----------
|
1412
|
-
models: list[NIM]
|
1413
|
-
List of NIM containers running models in sidecars.
|
1414
|
-
backend: str
|
1415
|
-
Compute provider to run the NIM container.
|
1416
|
-
"""
|
1417
|
-
...
|
1418
|
-
|
1419
1427
|
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1420
1428
|
"""
|
1421
1429
|
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
@@ -1458,55 +1466,6 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1458
1466
|
"""
|
1459
1467
|
...
|
1460
1468
|
|
1461
|
-
@typing.overload
|
1462
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1463
|
-
"""
|
1464
|
-
Specifies the Conda environment for all steps of the flow.
|
1465
|
-
|
1466
|
-
Use `@conda_base` to set common libraries required by all
|
1467
|
-
steps and use `@conda` to specify step-specific additions.
|
1468
|
-
|
1469
|
-
Parameters
|
1470
|
-
----------
|
1471
|
-
packages : Dict[str, str], default {}
|
1472
|
-
Packages to use for this flow. The key is the name of the package
|
1473
|
-
and the value is the version to use.
|
1474
|
-
libraries : Dict[str, str], default {}
|
1475
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1476
|
-
python : str, optional, default None
|
1477
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1478
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1479
|
-
disabled : bool, default False
|
1480
|
-
If set to True, disables Conda.
|
1481
|
-
"""
|
1482
|
-
...
|
1483
|
-
|
1484
|
-
@typing.overload
|
1485
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1486
|
-
...
|
1487
|
-
|
1488
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1489
|
-
"""
|
1490
|
-
Specifies the Conda environment for all steps of the flow.
|
1491
|
-
|
1492
|
-
Use `@conda_base` to set common libraries required by all
|
1493
|
-
steps and use `@conda` to specify step-specific additions.
|
1494
|
-
|
1495
|
-
Parameters
|
1496
|
-
----------
|
1497
|
-
packages : Dict[str, str], default {}
|
1498
|
-
Packages to use for this flow. The key is the name of the package
|
1499
|
-
and the value is the version to use.
|
1500
|
-
libraries : Dict[str, str], default {}
|
1501
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1502
|
-
python : str, optional, default None
|
1503
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1504
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1505
|
-
disabled : bool, default False
|
1506
|
-
If set to True, disables Conda.
|
1507
|
-
"""
|
1508
|
-
...
|
1509
|
-
|
1510
1469
|
@typing.overload
|
1511
1470
|
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1512
1471
|
"""
|
@@ -1602,55 +1561,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1602
1561
|
"""
|
1603
1562
|
...
|
1604
1563
|
|
1605
|
-
@typing.overload
|
1606
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1607
|
-
"""
|
1608
|
-
Specifies the times when the flow should be run when running on a
|
1609
|
-
production scheduler.
|
1610
|
-
|
1611
|
-
Parameters
|
1612
|
-
----------
|
1613
|
-
hourly : bool, default False
|
1614
|
-
Run the workflow hourly.
|
1615
|
-
daily : bool, default True
|
1616
|
-
Run the workflow daily.
|
1617
|
-
weekly : bool, default False
|
1618
|
-
Run the workflow weekly.
|
1619
|
-
cron : str, optional, default None
|
1620
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1621
|
-
specified by this expression.
|
1622
|
-
timezone : str, optional, default None
|
1623
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1624
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1625
|
-
"""
|
1626
|
-
...
|
1627
|
-
|
1628
|
-
@typing.overload
|
1629
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1630
|
-
...
|
1631
|
-
|
1632
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1633
|
-
"""
|
1634
|
-
Specifies the times when the flow should be run when running on a
|
1635
|
-
production scheduler.
|
1636
|
-
|
1637
|
-
Parameters
|
1638
|
-
----------
|
1639
|
-
hourly : bool, default False
|
1640
|
-
Run the workflow hourly.
|
1641
|
-
daily : bool, default True
|
1642
|
-
Run the workflow daily.
|
1643
|
-
weekly : bool, default False
|
1644
|
-
Run the workflow weekly.
|
1645
|
-
cron : str, optional, default None
|
1646
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1647
|
-
specified by this expression.
|
1648
|
-
timezone : str, optional, default None
|
1649
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1650
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1651
|
-
"""
|
1652
|
-
...
|
1653
|
-
|
1654
1564
|
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1655
1565
|
"""
|
1656
1566
|
Specifies what flows belong to the same project.
|
@@ -1711,42 +1621,132 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1711
1621
|
"""
|
1712
1622
|
...
|
1713
1623
|
|
1624
|
+
def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1625
|
+
"""
|
1626
|
+
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
1627
|
+
|
1628
|
+
User code call
|
1629
|
+
-----------
|
1630
|
+
@nim(
|
1631
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
1632
|
+
backend='managed'
|
1633
|
+
)
|
1634
|
+
|
1635
|
+
Valid backend options
|
1636
|
+
---------------------
|
1637
|
+
- 'managed': Outerbounds selects a compute provider based on the model.
|
1638
|
+
- 🚧 'dataplane': Run in your account.
|
1639
|
+
|
1640
|
+
Valid model options
|
1641
|
+
----------------
|
1642
|
+
- 'meta/llama3-8b-instruct': 8B parameter model
|
1643
|
+
- 'meta/llama3-70b-instruct': 70B parameter model
|
1644
|
+
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
1645
|
+
|
1646
|
+
Parameters
|
1647
|
+
----------
|
1648
|
+
models: list[NIM]
|
1649
|
+
List of NIM containers running models in sidecars.
|
1650
|
+
backend: str
|
1651
|
+
Compute provider to run the NIM container.
|
1652
|
+
"""
|
1653
|
+
...
|
1654
|
+
|
1714
1655
|
@typing.overload
|
1715
|
-
def
|
1656
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1716
1657
|
"""
|
1717
|
-
Specifies the
|
1658
|
+
Specifies the times when the flow should be run when running on a
|
1659
|
+
production scheduler.
|
1718
1660
|
|
1719
|
-
Use `@pypi_base` to set common packages required by all
|
1720
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1721
1661
|
Parameters
|
1722
1662
|
----------
|
1723
|
-
|
1663
|
+
hourly : bool, default False
|
1664
|
+
Run the workflow hourly.
|
1665
|
+
daily : bool, default True
|
1666
|
+
Run the workflow daily.
|
1667
|
+
weekly : bool, default False
|
1668
|
+
Run the workflow weekly.
|
1669
|
+
cron : str, optional, default None
|
1670
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1671
|
+
specified by this expression.
|
1672
|
+
timezone : str, optional, default None
|
1673
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1674
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1675
|
+
"""
|
1676
|
+
...
|
1677
|
+
|
1678
|
+
@typing.overload
|
1679
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1680
|
+
...
|
1681
|
+
|
1682
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1683
|
+
"""
|
1684
|
+
Specifies the times when the flow should be run when running on a
|
1685
|
+
production scheduler.
|
1686
|
+
|
1687
|
+
Parameters
|
1688
|
+
----------
|
1689
|
+
hourly : bool, default False
|
1690
|
+
Run the workflow hourly.
|
1691
|
+
daily : bool, default True
|
1692
|
+
Run the workflow daily.
|
1693
|
+
weekly : bool, default False
|
1694
|
+
Run the workflow weekly.
|
1695
|
+
cron : str, optional, default None
|
1696
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1697
|
+
specified by this expression.
|
1698
|
+
timezone : str, optional, default None
|
1699
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1700
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1701
|
+
"""
|
1702
|
+
...
|
1703
|
+
|
1704
|
+
@typing.overload
|
1705
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1706
|
+
"""
|
1707
|
+
Specifies the Conda environment for all steps of the flow.
|
1708
|
+
|
1709
|
+
Use `@conda_base` to set common libraries required by all
|
1710
|
+
steps and use `@conda` to specify step-specific additions.
|
1711
|
+
|
1712
|
+
Parameters
|
1713
|
+
----------
|
1714
|
+
packages : Dict[str, str], default {}
|
1724
1715
|
Packages to use for this flow. The key is the name of the package
|
1725
1716
|
and the value is the version to use.
|
1726
|
-
|
1717
|
+
libraries : Dict[str, str], default {}
|
1718
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1719
|
+
python : str, optional, default None
|
1727
1720
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1728
1721
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1722
|
+
disabled : bool, default False
|
1723
|
+
If set to True, disables Conda.
|
1729
1724
|
"""
|
1730
1725
|
...
|
1731
1726
|
|
1732
1727
|
@typing.overload
|
1733
|
-
def
|
1728
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1734
1729
|
...
|
1735
1730
|
|
1736
|
-
def
|
1731
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1737
1732
|
"""
|
1738
|
-
Specifies the
|
1733
|
+
Specifies the Conda environment for all steps of the flow.
|
1734
|
+
|
1735
|
+
Use `@conda_base` to set common libraries required by all
|
1736
|
+
steps and use `@conda` to specify step-specific additions.
|
1739
1737
|
|
1740
|
-
Use `@pypi_base` to set common packages required by all
|
1741
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1742
1738
|
Parameters
|
1743
1739
|
----------
|
1744
|
-
packages : Dict[str, str], default
|
1740
|
+
packages : Dict[str, str], default {}
|
1745
1741
|
Packages to use for this flow. The key is the name of the package
|
1746
1742
|
and the value is the version to use.
|
1747
|
-
|
1743
|
+
libraries : Dict[str, str], default {}
|
1744
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1745
|
+
python : str, optional, default None
|
1748
1746
|
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1749
1747
|
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1748
|
+
disabled : bool, default False
|
1749
|
+
If set to True, disables Conda.
|
1750
1750
|
"""
|
1751
1751
|
...
|
1752
1752
|
|