ob-metaflow-stubs 4.7__py2.py3-none-any.whl → 4.9__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +668 -619
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +2 -2
- metaflow-stubs/client/__init__.pyi +4 -4
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +4 -4
- metaflow-stubs/metadata/metadata.pyi +3 -3
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +35 -12
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +4 -4
- metaflow-stubs/plugins/__init__.pyi +6 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow.pyi +4 -4
- metaflow-stubs/plugins/airflow/airflow_cli.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_decorator.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +38 -5
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +269 -4
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +229 -4
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +292 -0
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_cli.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/dynamo_db_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_cli.pyi +23 -3
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +260 -0
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +5 -5
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/main.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/metadata.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/renderer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/chevron/tokenizer.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +2 -2
- metaflow-stubs/plugins/datatools/__init__.pyi +4 -4
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +4 -4
- metaflow-stubs/plugins/datatools/s3/s3.pyi +6 -6
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +2 -2
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +4 -2
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +18 -10
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +12 -12
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +14 -31
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +82 -28
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +47 -2
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +3 -3
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +4 -4
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +26 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +354 -0
- metaflow-stubs/runner/metaflow_runner.pyi +11 -11
- metaflow-stubs/runner/nbdeploy.pyi +54 -0
- metaflow-stubs/runner/nbrun.pyi +7 -7
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +25 -0
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +2 -2
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-4.7.dist-info → ob_metaflow_stubs-4.9.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-4.9.dist-info/RECORD +150 -0
- ob_metaflow_stubs-4.7.dist-info/RECORD +0 -145
- {ob_metaflow_stubs-4.7.dist-info → ob_metaflow_stubs-4.9.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-4.7.dist-info → ob_metaflow_stubs-4.9.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,25 +1,25 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-07-
|
3
|
+
# MF version: 2.12.10.1+ob(v1) #
|
4
|
+
# Generated on 2024-07-31T05:43:20.962506 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import
|
12
|
-
import
|
11
|
+
import metaflow.events
|
12
|
+
import metaflow.flowspec
|
13
|
+
import metaflow.metaflow_current
|
14
|
+
import metaflow._vendor.click.types
|
13
15
|
import metaflow.plugins.datatools.s3.s3
|
14
|
-
import metaflow.
|
16
|
+
import metaflow.client.core
|
15
17
|
import metaflow.datastore.inputs
|
18
|
+
import datetime
|
16
19
|
import metaflow.runner.metaflow_runner
|
20
|
+
import typing
|
21
|
+
import metaflow.parameters
|
17
22
|
import io
|
18
|
-
import metaflow.metaflow_current
|
19
|
-
import metaflow.flowspec
|
20
|
-
import metaflow._vendor.click.types
|
21
|
-
import metaflow.client.core
|
22
|
-
import metaflow.events
|
23
23
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
24
24
|
StepFlag = typing.NewType("StepFlag", bool)
|
25
25
|
|
@@ -728,279 +728,141 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
728
728
|
...
|
729
729
|
|
730
730
|
@typing.overload
|
731
|
-
def
|
731
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
732
732
|
"""
|
733
|
-
Specifies
|
734
|
-
to a step needs to be retried.
|
735
|
-
|
736
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
737
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
738
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
739
|
-
|
740
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
741
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
742
|
-
ensuring that the flow execution can continue.
|
733
|
+
Specifies environment variables to be set prior to the execution of a step.
|
743
734
|
|
744
735
|
Parameters
|
745
736
|
----------
|
746
|
-
|
747
|
-
|
748
|
-
minutes_between_retries : int, default 2
|
749
|
-
Number of minutes between retries.
|
737
|
+
vars : Dict[str, str], default {}
|
738
|
+
Dictionary of environment variables to set.
|
750
739
|
"""
|
751
740
|
...
|
752
741
|
|
753
742
|
@typing.overload
|
754
|
-
def
|
743
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
755
744
|
...
|
756
745
|
|
757
746
|
@typing.overload
|
758
|
-
def
|
747
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
759
748
|
...
|
760
749
|
|
761
|
-
def
|
750
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
762
751
|
"""
|
763
|
-
Specifies
|
764
|
-
to a step needs to be retried.
|
765
|
-
|
766
|
-
This decorator is useful for handling transient errors, such as networking issues.
|
767
|
-
If your task contains operations that can't be retried safely, e.g. database updates,
|
768
|
-
it is advisable to annotate it with `@retry(times=0)`.
|
769
|
-
|
770
|
-
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
771
|
-
decorator will execute a no-op task after all retries have been exhausted,
|
772
|
-
ensuring that the flow execution can continue.
|
752
|
+
Specifies environment variables to be set prior to the execution of a step.
|
773
753
|
|
774
754
|
Parameters
|
775
755
|
----------
|
776
|
-
|
777
|
-
|
778
|
-
minutes_between_retries : int, default 2
|
779
|
-
Number of minutes between retries.
|
756
|
+
vars : Dict[str, str], default {}
|
757
|
+
Dictionary of environment variables to set.
|
780
758
|
"""
|
781
759
|
...
|
782
760
|
|
783
761
|
@typing.overload
|
784
|
-
def
|
762
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
785
763
|
"""
|
786
|
-
Specifies the
|
787
|
-
|
788
|
-
Use `@resources` to specify the resource requirements
|
789
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
764
|
+
Specifies the Conda environment for the step.
|
790
765
|
|
791
|
-
|
792
|
-
|
793
|
-
|
794
|
-
|
795
|
-
or
|
796
|
-
```
|
797
|
-
python myflow.py run --with kubernetes
|
798
|
-
```
|
799
|
-
which executes the flow on the desired system using the
|
800
|
-
requirements specified in `@resources`.
|
766
|
+
Information in this decorator will augment any
|
767
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
768
|
+
you can use `@conda_base` to set packages required by all
|
769
|
+
steps and use `@conda` to specify step-specific overrides.
|
801
770
|
|
802
771
|
Parameters
|
803
772
|
----------
|
804
|
-
|
805
|
-
|
806
|
-
|
807
|
-
|
808
|
-
|
809
|
-
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
|
814
|
-
This parameter maps to the `--shm-size` option in Docker.
|
773
|
+
packages : Dict[str, str], default {}
|
774
|
+
Packages to use for this step. The key is the name of the package
|
775
|
+
and the value is the version to use.
|
776
|
+
libraries : Dict[str, str], default {}
|
777
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
778
|
+
python : str, optional, default None
|
779
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
780
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
781
|
+
disabled : bool, default False
|
782
|
+
If set to True, disables @conda.
|
815
783
|
"""
|
816
784
|
...
|
817
785
|
|
818
786
|
@typing.overload
|
819
|
-
def
|
787
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
820
788
|
...
|
821
789
|
|
822
790
|
@typing.overload
|
823
|
-
def
|
791
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
824
792
|
...
|
825
793
|
|
826
|
-
def
|
794
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
827
795
|
"""
|
828
|
-
Specifies the
|
829
|
-
|
830
|
-
Use `@resources` to specify the resource requirements
|
831
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
796
|
+
Specifies the Conda environment for the step.
|
832
797
|
|
833
|
-
|
834
|
-
|
835
|
-
|
836
|
-
|
837
|
-
or
|
838
|
-
```
|
839
|
-
python myflow.py run --with kubernetes
|
840
|
-
```
|
841
|
-
which executes the flow on the desired system using the
|
842
|
-
requirements specified in `@resources`.
|
798
|
+
Information in this decorator will augment any
|
799
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
800
|
+
you can use `@conda_base` to set packages required by all
|
801
|
+
steps and use `@conda` to specify step-specific overrides.
|
843
802
|
|
844
803
|
Parameters
|
845
804
|
----------
|
846
|
-
|
847
|
-
|
848
|
-
|
849
|
-
|
850
|
-
|
851
|
-
|
852
|
-
|
853
|
-
|
854
|
-
|
855
|
-
|
856
|
-
This parameter maps to the `--shm-size` option in Docker.
|
805
|
+
packages : Dict[str, str], default {}
|
806
|
+
Packages to use for this step. The key is the name of the package
|
807
|
+
and the value is the version to use.
|
808
|
+
libraries : Dict[str, str], default {}
|
809
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
810
|
+
python : str, optional, default None
|
811
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
812
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
813
|
+
disabled : bool, default False
|
814
|
+
If set to True, disables @conda.
|
857
815
|
"""
|
858
816
|
...
|
859
817
|
|
860
818
|
@typing.overload
|
861
|
-
def
|
819
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
862
820
|
"""
|
863
|
-
|
821
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
822
|
+
|
823
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
864
824
|
|
865
825
|
Parameters
|
866
826
|
----------
|
867
|
-
|
868
|
-
|
869
|
-
|
870
|
-
|
871
|
-
|
872
|
-
|
873
|
-
|
874
|
-
|
875
|
-
|
876
|
-
|
877
|
-
image : str, optional, default None
|
878
|
-
Docker image to use when launching on AWS Batch. If not specified, and
|
879
|
-
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
880
|
-
not, a default Docker image mapping to the current version of Python is used.
|
881
|
-
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
882
|
-
AWS Batch Job Queue to submit the job to.
|
883
|
-
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
884
|
-
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
885
|
-
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
886
|
-
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
887
|
-
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
888
|
-
shared_memory : int, optional, default None
|
889
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
890
|
-
This parameter maps to the `--shm-size` option in Docker.
|
891
|
-
max_swap : int, optional, default None
|
892
|
-
The total amount of swap memory (in MiB) a container can use for this
|
893
|
-
step. This parameter is translated to the `--memory-swap` option in
|
894
|
-
Docker where the value is the sum of the container memory plus the
|
895
|
-
`max_swap` value.
|
896
|
-
swappiness : int, optional, default None
|
897
|
-
This allows you to tune memory swappiness behavior for this step.
|
898
|
-
A swappiness value of 0 causes swapping not to happen unless absolutely
|
899
|
-
necessary. A swappiness value of 100 causes pages to be swapped very
|
900
|
-
aggressively. Accepted values are whole numbers between 0 and 100.
|
901
|
-
use_tmpfs : bool, default False
|
902
|
-
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
903
|
-
not available on Fargate compute environments
|
904
|
-
tmpfs_tempdir : bool, default True
|
905
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
906
|
-
tmpfs_size : int, optional, default None
|
907
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
908
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
909
|
-
memory allocated for this step.
|
910
|
-
tmpfs_path : str, optional, default None
|
911
|
-
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
912
|
-
inferentia : int, default 0
|
913
|
-
Number of Inferentia chips required for this step.
|
914
|
-
trainium : int, default None
|
915
|
-
Alias for inferentia. Use only one of the two.
|
916
|
-
efa : int, default 0
|
917
|
-
Number of elastic fabric adapter network devices to attach to container
|
918
|
-
ephemeral_storage : int, default None
|
919
|
-
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
920
|
-
This is only relevant for Fargate compute environments
|
921
|
-
log_driver: str, optional, default None
|
922
|
-
The log driver to use for the Amazon ECS container.
|
923
|
-
log_options: List[str], optional, default None
|
924
|
-
List of strings containing options for the chosen log driver. The configurable values
|
925
|
-
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
926
|
-
Example: [`awslogs-group:aws/batch/job`]
|
827
|
+
type : str, default 'default'
|
828
|
+
Card type.
|
829
|
+
id : str, optional, default None
|
830
|
+
If multiple cards are present, use this id to identify this card.
|
831
|
+
options : Dict[str, Any], default {}
|
832
|
+
Options passed to the card. The contents depend on the card type.
|
833
|
+
timeout : int, default 45
|
834
|
+
Interrupt reporting if it takes more than this many seconds.
|
835
|
+
|
836
|
+
|
927
837
|
"""
|
928
838
|
...
|
929
839
|
|
930
840
|
@typing.overload
|
931
|
-
def
|
841
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
932
842
|
...
|
933
843
|
|
934
844
|
@typing.overload
|
935
|
-
def
|
845
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
936
846
|
...
|
937
847
|
|
938
|
-
def
|
848
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
939
849
|
"""
|
940
|
-
|
850
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
851
|
+
|
852
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
941
853
|
|
942
854
|
Parameters
|
943
855
|
----------
|
944
|
-
|
945
|
-
|
946
|
-
|
947
|
-
|
948
|
-
|
949
|
-
|
950
|
-
|
951
|
-
|
952
|
-
|
953
|
-
|
954
|
-
image : str, optional, default None
|
955
|
-
Docker image to use when launching on AWS Batch. If not specified, and
|
956
|
-
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
957
|
-
not, a default Docker image mapping to the current version of Python is used.
|
958
|
-
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
959
|
-
AWS Batch Job Queue to submit the job to.
|
960
|
-
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
961
|
-
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
962
|
-
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
963
|
-
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
964
|
-
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
965
|
-
shared_memory : int, optional, default None
|
966
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
967
|
-
This parameter maps to the `--shm-size` option in Docker.
|
968
|
-
max_swap : int, optional, default None
|
969
|
-
The total amount of swap memory (in MiB) a container can use for this
|
970
|
-
step. This parameter is translated to the `--memory-swap` option in
|
971
|
-
Docker where the value is the sum of the container memory plus the
|
972
|
-
`max_swap` value.
|
973
|
-
swappiness : int, optional, default None
|
974
|
-
This allows you to tune memory swappiness behavior for this step.
|
975
|
-
A swappiness value of 0 causes swapping not to happen unless absolutely
|
976
|
-
necessary. A swappiness value of 100 causes pages to be swapped very
|
977
|
-
aggressively. Accepted values are whole numbers between 0 and 100.
|
978
|
-
use_tmpfs : bool, default False
|
979
|
-
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
980
|
-
not available on Fargate compute environments
|
981
|
-
tmpfs_tempdir : bool, default True
|
982
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
983
|
-
tmpfs_size : int, optional, default None
|
984
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
985
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
986
|
-
memory allocated for this step.
|
987
|
-
tmpfs_path : str, optional, default None
|
988
|
-
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
989
|
-
inferentia : int, default 0
|
990
|
-
Number of Inferentia chips required for this step.
|
991
|
-
trainium : int, default None
|
992
|
-
Alias for inferentia. Use only one of the two.
|
993
|
-
efa : int, default 0
|
994
|
-
Number of elastic fabric adapter network devices to attach to container
|
995
|
-
ephemeral_storage : int, default None
|
996
|
-
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
997
|
-
This is only relevant for Fargate compute environments
|
998
|
-
log_driver: str, optional, default None
|
999
|
-
The log driver to use for the Amazon ECS container.
|
1000
|
-
log_options: List[str], optional, default None
|
1001
|
-
List of strings containing options for the chosen log driver. The configurable values
|
1002
|
-
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
1003
|
-
Example: [`awslogs-group:aws/batch/job`]
|
856
|
+
type : str, default 'default'
|
857
|
+
Card type.
|
858
|
+
id : str, optional, default None
|
859
|
+
If multiple cards are present, use this id to identify this card.
|
860
|
+
options : Dict[str, Any], default {}
|
861
|
+
Options passed to the card. The contents depend on the card type.
|
862
|
+
timeout : int, default 45
|
863
|
+
Interrupt reporting if it takes more than this many seconds.
|
864
|
+
|
865
|
+
|
1004
866
|
"""
|
1005
867
|
...
|
1006
868
|
|
@@ -1061,6 +923,55 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
1061
923
|
"""
|
1062
924
|
...
|
1063
925
|
|
926
|
+
@typing.overload
|
927
|
+
def catch(*, var: typing.Optional[str] = None, print_exception: bool = True) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
928
|
+
"""
|
929
|
+
Specifies that the step will success under all circumstances.
|
930
|
+
|
931
|
+
The decorator will create an optional artifact, specified by `var`, which
|
932
|
+
contains the exception raised. You can use it to detect the presence
|
933
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
934
|
+
are missing.
|
935
|
+
|
936
|
+
Parameters
|
937
|
+
----------
|
938
|
+
var : str, optional, default None
|
939
|
+
Name of the artifact in which to store the caught exception.
|
940
|
+
If not specified, the exception is not stored.
|
941
|
+
print_exception : bool, default True
|
942
|
+
Determines whether or not the exception is printed to
|
943
|
+
stdout when caught.
|
944
|
+
"""
|
945
|
+
...
|
946
|
+
|
947
|
+
@typing.overload
|
948
|
+
def catch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
949
|
+
...
|
950
|
+
|
951
|
+
@typing.overload
|
952
|
+
def catch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
953
|
+
...
|
954
|
+
|
955
|
+
def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, var: typing.Optional[str] = None, print_exception: bool = True):
|
956
|
+
"""
|
957
|
+
Specifies that the step will success under all circumstances.
|
958
|
+
|
959
|
+
The decorator will create an optional artifact, specified by `var`, which
|
960
|
+
contains the exception raised. You can use it to detect the presence
|
961
|
+
of errors, indicating that all happy-path artifacts produced by the step
|
962
|
+
are missing.
|
963
|
+
|
964
|
+
Parameters
|
965
|
+
----------
|
966
|
+
var : str, optional, default None
|
967
|
+
Name of the artifact in which to store the caught exception.
|
968
|
+
If not specified, the exception is not stored.
|
969
|
+
print_exception : bool, default True
|
970
|
+
Determines whether or not the exception is printed to
|
971
|
+
stdout when caught.
|
972
|
+
"""
|
973
|
+
...
|
974
|
+
|
1064
975
|
@typing.overload
|
1065
976
|
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1066
977
|
"""
|
@@ -1094,6 +1005,78 @@ def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
1094
1005
|
"""
|
1095
1006
|
...
|
1096
1007
|
|
1008
|
+
@typing.overload
|
1009
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1010
|
+
"""
|
1011
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1012
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1013
|
+
"""
|
1014
|
+
...
|
1015
|
+
|
1016
|
+
@typing.overload
|
1017
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1018
|
+
...
|
1019
|
+
|
1020
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1021
|
+
"""
|
1022
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1023
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1024
|
+
"""
|
1025
|
+
...
|
1026
|
+
|
1027
|
+
@typing.overload
|
1028
|
+
def retry(*, times: int = 3, minutes_between_retries: int = 2) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1029
|
+
"""
|
1030
|
+
Specifies the number of times the task corresponding
|
1031
|
+
to a step needs to be retried.
|
1032
|
+
|
1033
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
1034
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
1035
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
1036
|
+
|
1037
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
1038
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
1039
|
+
ensuring that the flow execution can continue.
|
1040
|
+
|
1041
|
+
Parameters
|
1042
|
+
----------
|
1043
|
+
times : int, default 3
|
1044
|
+
Number of times to retry this task.
|
1045
|
+
minutes_between_retries : int, default 2
|
1046
|
+
Number of minutes between retries.
|
1047
|
+
"""
|
1048
|
+
...
|
1049
|
+
|
1050
|
+
@typing.overload
|
1051
|
+
def retry(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1052
|
+
...
|
1053
|
+
|
1054
|
+
@typing.overload
|
1055
|
+
def retry(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1056
|
+
...
|
1057
|
+
|
1058
|
+
def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, times: int = 3, minutes_between_retries: int = 2):
|
1059
|
+
"""
|
1060
|
+
Specifies the number of times the task corresponding
|
1061
|
+
to a step needs to be retried.
|
1062
|
+
|
1063
|
+
This decorator is useful for handling transient errors, such as networking issues.
|
1064
|
+
If your task contains operations that can't be retried safely, e.g. database updates,
|
1065
|
+
it is advisable to annotate it with `@retry(times=0)`.
|
1066
|
+
|
1067
|
+
This can be used in conjunction with the `@catch` decorator. The `@catch`
|
1068
|
+
decorator will execute a no-op task after all retries have been exhausted,
|
1069
|
+
ensuring that the flow execution can continue.
|
1070
|
+
|
1071
|
+
Parameters
|
1072
|
+
----------
|
1073
|
+
times : int, default 3
|
1074
|
+
Number of times to retry this task.
|
1075
|
+
minutes_between_retries : int, default 2
|
1076
|
+
Number of minutes between retries.
|
1077
|
+
"""
|
1078
|
+
...
|
1079
|
+
|
1097
1080
|
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1098
1081
|
"""
|
1099
1082
|
Specifies that this step should execute on Kubernetes.
|
@@ -1154,108 +1137,149 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
1154
1137
|
...
|
1155
1138
|
|
1156
1139
|
@typing.overload
|
1157
|
-
def
|
1140
|
+
def batch(*, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1158
1141
|
"""
|
1159
|
-
Specifies
|
1160
|
-
|
1161
|
-
Information in this decorator will augment any
|
1162
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1163
|
-
you can use `@conda_base` to set packages required by all
|
1164
|
-
steps and use `@conda` to specify step-specific overrides.
|
1142
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
1165
1143
|
|
1166
1144
|
Parameters
|
1167
1145
|
----------
|
1168
|
-
|
1169
|
-
|
1170
|
-
|
1171
|
-
|
1172
|
-
|
1173
|
-
|
1174
|
-
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1146
|
+
cpu : int, default 1
|
1147
|
+
Number of CPUs required for this step. If `@resources` is
|
1148
|
+
also present, the maximum value from all decorators is used.
|
1149
|
+
gpu : int, default 0
|
1150
|
+
Number of GPUs required for this step. If `@resources` is
|
1151
|
+
also present, the maximum value from all decorators is used.
|
1152
|
+
memory : int, default 4096
|
1153
|
+
Memory size (in MB) required for this step. If
|
1154
|
+
`@resources` is also present, the maximum value from all decorators is
|
1155
|
+
used.
|
1156
|
+
image : str, optional, default None
|
1157
|
+
Docker image to use when launching on AWS Batch. If not specified, and
|
1158
|
+
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
1159
|
+
not, a default Docker image mapping to the current version of Python is used.
|
1160
|
+
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
1161
|
+
AWS Batch Job Queue to submit the job to.
|
1162
|
+
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
1163
|
+
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
1164
|
+
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
1165
|
+
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
1166
|
+
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
1167
|
+
shared_memory : int, optional, default None
|
1168
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1169
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1170
|
+
max_swap : int, optional, default None
|
1171
|
+
The total amount of swap memory (in MiB) a container can use for this
|
1172
|
+
step. This parameter is translated to the `--memory-swap` option in
|
1173
|
+
Docker where the value is the sum of the container memory plus the
|
1174
|
+
`max_swap` value.
|
1175
|
+
swappiness : int, optional, default None
|
1176
|
+
This allows you to tune memory swappiness behavior for this step.
|
1177
|
+
A swappiness value of 0 causes swapping not to happen unless absolutely
|
1178
|
+
necessary. A swappiness value of 100 causes pages to be swapped very
|
1179
|
+
aggressively. Accepted values are whole numbers between 0 and 100.
|
1180
|
+
use_tmpfs : bool, default False
|
1181
|
+
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
1182
|
+
not available on Fargate compute environments
|
1183
|
+
tmpfs_tempdir : bool, default True
|
1184
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
1185
|
+
tmpfs_size : int, optional, default None
|
1186
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
1187
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
1188
|
+
memory allocated for this step.
|
1189
|
+
tmpfs_path : str, optional, default None
|
1190
|
+
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
1191
|
+
inferentia : int, default 0
|
1192
|
+
Number of Inferentia chips required for this step.
|
1193
|
+
trainium : int, default None
|
1194
|
+
Alias for inferentia. Use only one of the two.
|
1195
|
+
efa : int, default 0
|
1196
|
+
Number of elastic fabric adapter network devices to attach to container
|
1197
|
+
ephemeral_storage : int, default None
|
1198
|
+
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
1199
|
+
This is only relevant for Fargate compute environments
|
1200
|
+
log_driver: str, optional, default None
|
1201
|
+
The log driver to use for the Amazon ECS container.
|
1202
|
+
log_options: List[str], optional, default None
|
1203
|
+
List of strings containing options for the chosen log driver. The configurable values
|
1204
|
+
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
1205
|
+
Example: [`awslogs-group:aws/batch/job`]
|
1178
1206
|
"""
|
1179
1207
|
...
|
1180
1208
|
|
1181
1209
|
@typing.overload
|
1182
|
-
def
|
1210
|
+
def batch(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1183
1211
|
...
|
1184
1212
|
|
1185
1213
|
@typing.overload
|
1186
|
-
def
|
1214
|
+
def batch(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1187
1215
|
...
|
1188
1216
|
|
1189
|
-
def
|
1217
|
+
def batch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, memory: int = 4096, image: typing.Optional[str] = None, queue: str = "METAFLOW_BATCH_JOB_QUEUE", iam_role: str = "METAFLOW_ECS_S3_ACCESS_IAM_ROLE", execution_role: str = "METAFLOW_ECS_FARGATE_EXECUTION_ROLE", shared_memory: typing.Optional[int] = None, max_swap: typing.Optional[int] = None, swappiness: typing.Optional[int] = None, use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = None, inferentia: int = 0, trainium: int = None, efa: int = 0, ephemeral_storage: int = None, log_driver: typing.Optional[str] = None, log_options: typing.Optional[typing.List[str]] = None):
|
1190
1218
|
"""
|
1191
|
-
Specifies
|
1192
|
-
|
1193
|
-
Information in this decorator will augment any
|
1194
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1195
|
-
you can use `@conda_base` to set packages required by all
|
1196
|
-
steps and use `@conda` to specify step-specific overrides.
|
1219
|
+
Specifies that this step should execute on [AWS Batch](https://aws.amazon.com/batch/).
|
1197
1220
|
|
1198
1221
|
Parameters
|
1199
1222
|
----------
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1206
|
-
|
1207
|
-
|
1208
|
-
|
1209
|
-
|
1210
|
-
|
1211
|
-
|
1212
|
-
|
1213
|
-
|
1214
|
-
|
1215
|
-
|
1216
|
-
|
1217
|
-
|
1218
|
-
|
1219
|
-
|
1220
|
-
|
1221
|
-
|
1222
|
-
|
1223
|
-
|
1224
|
-
|
1225
|
-
|
1226
|
-
|
1227
|
-
|
1228
|
-
|
1229
|
-
|
1230
|
-
|
1231
|
-
|
1232
|
-
|
1233
|
-
|
1234
|
-
|
1235
|
-
|
1236
|
-
|
1237
|
-
|
1238
|
-
|
1239
|
-
|
1240
|
-
|
1241
|
-
|
1242
|
-
|
1243
|
-
|
1244
|
-
|
1245
|
-
|
1246
|
-
|
1247
|
-
|
1248
|
-
|
1249
|
-
|
1250
|
-
|
1251
|
-
|
1252
|
-
|
1253
|
-
|
1254
|
-
|
1255
|
-
|
1256
|
-
|
1257
|
-
|
1258
|
-
|
1223
|
+
cpu : int, default 1
|
1224
|
+
Number of CPUs required for this step. If `@resources` is
|
1225
|
+
also present, the maximum value from all decorators is used.
|
1226
|
+
gpu : int, default 0
|
1227
|
+
Number of GPUs required for this step. If `@resources` is
|
1228
|
+
also present, the maximum value from all decorators is used.
|
1229
|
+
memory : int, default 4096
|
1230
|
+
Memory size (in MB) required for this step. If
|
1231
|
+
`@resources` is also present, the maximum value from all decorators is
|
1232
|
+
used.
|
1233
|
+
image : str, optional, default None
|
1234
|
+
Docker image to use when launching on AWS Batch. If not specified, and
|
1235
|
+
METAFLOW_BATCH_CONTAINER_IMAGE is specified, that image is used. If
|
1236
|
+
not, a default Docker image mapping to the current version of Python is used.
|
1237
|
+
queue : str, default METAFLOW_BATCH_JOB_QUEUE
|
1238
|
+
AWS Batch Job Queue to submit the job to.
|
1239
|
+
iam_role : str, default METAFLOW_ECS_S3_ACCESS_IAM_ROLE
|
1240
|
+
AWS IAM role that AWS Batch container uses to access AWS cloud resources.
|
1241
|
+
execution_role : str, default METAFLOW_ECS_FARGATE_EXECUTION_ROLE
|
1242
|
+
AWS IAM role that AWS Batch can use [to trigger AWS Fargate tasks]
|
1243
|
+
(https://docs.aws.amazon.com/batch/latest/userguide/execution-IAM-role.html).
|
1244
|
+
shared_memory : int, optional, default None
|
1245
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1246
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1247
|
+
max_swap : int, optional, default None
|
1248
|
+
The total amount of swap memory (in MiB) a container can use for this
|
1249
|
+
step. This parameter is translated to the `--memory-swap` option in
|
1250
|
+
Docker where the value is the sum of the container memory plus the
|
1251
|
+
`max_swap` value.
|
1252
|
+
swappiness : int, optional, default None
|
1253
|
+
This allows you to tune memory swappiness behavior for this step.
|
1254
|
+
A swappiness value of 0 causes swapping not to happen unless absolutely
|
1255
|
+
necessary. A swappiness value of 100 causes pages to be swapped very
|
1256
|
+
aggressively. Accepted values are whole numbers between 0 and 100.
|
1257
|
+
use_tmpfs : bool, default False
|
1258
|
+
This enables an explicit tmpfs mount for this step. Note that tmpfs is
|
1259
|
+
not available on Fargate compute environments
|
1260
|
+
tmpfs_tempdir : bool, default True
|
1261
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
1262
|
+
tmpfs_size : int, optional, default None
|
1263
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
1264
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
1265
|
+
memory allocated for this step.
|
1266
|
+
tmpfs_path : str, optional, default None
|
1267
|
+
Path to tmpfs mount for this step. Defaults to /metaflow_temp.
|
1268
|
+
inferentia : int, default 0
|
1269
|
+
Number of Inferentia chips required for this step.
|
1270
|
+
trainium : int, default None
|
1271
|
+
Alias for inferentia. Use only one of the two.
|
1272
|
+
efa : int, default 0
|
1273
|
+
Number of elastic fabric adapter network devices to attach to container
|
1274
|
+
ephemeral_storage : int, default None
|
1275
|
+
The total amount, in GiB, of ephemeral storage to set for the task, 21-200GiB.
|
1276
|
+
This is only relevant for Fargate compute environments
|
1277
|
+
log_driver: str, optional, default None
|
1278
|
+
The log driver to use for the Amazon ECS container.
|
1279
|
+
log_options: List[str], optional, default None
|
1280
|
+
List of strings containing options for the chosen log driver. The configurable values
|
1281
|
+
depend on the `log driver` chosen. Validation of these options is not supported yet.
|
1282
|
+
Example: [`awslogs-group:aws/batch/job`]
|
1259
1283
|
"""
|
1260
1284
|
...
|
1261
1285
|
|
@@ -1309,187 +1333,167 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
1309
1333
|
...
|
1310
1334
|
|
1311
1335
|
@typing.overload
|
1312
|
-
def
|
1336
|
+
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1313
1337
|
"""
|
1314
|
-
|
1338
|
+
Specifies the resources needed when executing this step.
|
1315
1339
|
|
1316
|
-
|
1340
|
+
Use `@resources` to specify the resource requirements
|
1341
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1342
|
+
|
1343
|
+
You can choose the compute layer on the command line by executing e.g.
|
1344
|
+
```
|
1345
|
+
python myflow.py run --with batch
|
1346
|
+
```
|
1347
|
+
or
|
1348
|
+
```
|
1349
|
+
python myflow.py run --with kubernetes
|
1350
|
+
```
|
1351
|
+
which executes the flow on the desired system using the
|
1352
|
+
requirements specified in `@resources`.
|
1317
1353
|
|
1318
1354
|
Parameters
|
1319
1355
|
----------
|
1320
|
-
|
1321
|
-
|
1322
|
-
|
1323
|
-
|
1324
|
-
|
1325
|
-
|
1326
|
-
|
1327
|
-
|
1328
|
-
|
1329
|
-
|
1356
|
+
cpu : int, default 1
|
1357
|
+
Number of CPUs required for this step.
|
1358
|
+
gpu : int, default 0
|
1359
|
+
Number of GPUs required for this step.
|
1360
|
+
disk : int, optional, default None
|
1361
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1362
|
+
memory : int, default 4096
|
1363
|
+
Memory size (in MB) required for this step.
|
1364
|
+
shared_memory : int, optional, default None
|
1365
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1366
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1330
1367
|
"""
|
1331
1368
|
...
|
1332
1369
|
|
1333
1370
|
@typing.overload
|
1334
|
-
def
|
1371
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1335
1372
|
...
|
1336
1373
|
|
1337
1374
|
@typing.overload
|
1338
|
-
def
|
1375
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1339
1376
|
...
|
1340
1377
|
|
1341
|
-
def
|
1378
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
1342
1379
|
"""
|
1343
|
-
|
1380
|
+
Specifies the resources needed when executing this step.
|
1344
1381
|
|
1345
|
-
|
1382
|
+
Use `@resources` to specify the resource requirements
|
1383
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1384
|
+
|
1385
|
+
You can choose the compute layer on the command line by executing e.g.
|
1386
|
+
```
|
1387
|
+
python myflow.py run --with batch
|
1388
|
+
```
|
1389
|
+
or
|
1390
|
+
```
|
1391
|
+
python myflow.py run --with kubernetes
|
1392
|
+
```
|
1393
|
+
which executes the flow on the desired system using the
|
1394
|
+
requirements specified in `@resources`.
|
1346
1395
|
|
1347
1396
|
Parameters
|
1348
1397
|
----------
|
1349
|
-
|
1350
|
-
|
1351
|
-
|
1352
|
-
|
1353
|
-
|
1354
|
-
|
1355
|
-
|
1356
|
-
|
1357
|
-
|
1358
|
-
|
1398
|
+
cpu : int, default 1
|
1399
|
+
Number of CPUs required for this step.
|
1400
|
+
gpu : int, default 0
|
1401
|
+
Number of GPUs required for this step.
|
1402
|
+
disk : int, optional, default None
|
1403
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1404
|
+
memory : int, default 4096
|
1405
|
+
Memory size (in MB) required for this step.
|
1406
|
+
shared_memory : int, optional, default None
|
1407
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1408
|
+
This parameter maps to the `--shm-size` option in Docker.
|
1359
1409
|
"""
|
1360
1410
|
...
|
1361
1411
|
|
1362
|
-
|
1363
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1412
|
+
def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1364
1413
|
"""
|
1365
|
-
|
1414
|
+
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
1415
|
+
|
1416
|
+
User code call
|
1417
|
+
-----------
|
1418
|
+
@nim(
|
1419
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
1420
|
+
backend='managed'
|
1421
|
+
)
|
1422
|
+
|
1423
|
+
Valid backend options
|
1424
|
+
---------------------
|
1425
|
+
- 'managed': Outerbounds selects a compute provider based on the model.
|
1426
|
+
- 🚧 'dataplane': Run in your account.
|
1427
|
+
|
1428
|
+
Valid model options
|
1429
|
+
----------------
|
1430
|
+
- 'meta/llama3-8b-instruct': 8B parameter model
|
1431
|
+
- 'meta/llama3-70b-instruct': 70B parameter model
|
1432
|
+
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
1366
1433
|
|
1367
1434
|
Parameters
|
1368
1435
|
----------
|
1369
|
-
|
1370
|
-
|
1436
|
+
models: list[NIM]
|
1437
|
+
List of NIM containers running models in sidecars.
|
1438
|
+
backend: str
|
1439
|
+
Compute provider to run the NIM container.
|
1371
1440
|
"""
|
1372
1441
|
...
|
1373
1442
|
|
1374
|
-
|
1375
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1376
|
-
...
|
1377
|
-
|
1378
|
-
@typing.overload
|
1379
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1380
|
-
...
|
1381
|
-
|
1382
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1443
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1383
1444
|
"""
|
1384
|
-
Specifies
|
1445
|
+
Specifies what flows belong to the same project.
|
1446
|
+
|
1447
|
+
A project-specific namespace is created for all flows that
|
1448
|
+
use the same `@project(name)`.
|
1385
1449
|
|
1386
1450
|
Parameters
|
1387
1451
|
----------
|
1388
|
-
|
1389
|
-
|
1452
|
+
name : str
|
1453
|
+
Project name. Make sure that the name is unique amongst all
|
1454
|
+
projects that use the same production scheduler. The name may
|
1455
|
+
contain only lowercase alphanumeric characters and underscores.
|
1456
|
+
|
1457
|
+
|
1390
1458
|
"""
|
1391
1459
|
...
|
1392
1460
|
|
1393
1461
|
@typing.overload
|
1394
|
-
def
|
1462
|
+
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1395
1463
|
"""
|
1396
|
-
Specifies the
|
1397
|
-
|
1398
|
-
```
|
1399
|
-
@trigger_on_finish(flow='FooFlow')
|
1400
|
-
```
|
1401
|
-
or
|
1402
|
-
```
|
1403
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1404
|
-
```
|
1405
|
-
This decorator respects the @project decorator and triggers the flow
|
1406
|
-
when upstream runs within the same namespace complete successfully
|
1407
|
-
|
1408
|
-
Additionally, you can specify project aware upstream flow dependencies
|
1409
|
-
by specifying the fully qualified project_flow_name.
|
1410
|
-
```
|
1411
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1412
|
-
```
|
1413
|
-
or
|
1414
|
-
```
|
1415
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1416
|
-
```
|
1417
|
-
|
1418
|
-
You can also specify just the project or project branch (other values will be
|
1419
|
-
inferred from the current project or project branch):
|
1420
|
-
```
|
1421
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1422
|
-
```
|
1423
|
-
|
1424
|
-
Note that `branch` is typically one of:
|
1425
|
-
- `prod`
|
1426
|
-
- `user.bob`
|
1427
|
-
- `test.my_experiment`
|
1428
|
-
- `prod.staging`
|
1464
|
+
Specifies the PyPI packages for all steps of the flow.
|
1429
1465
|
|
1466
|
+
Use `@pypi_base` to set common packages required by all
|
1467
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1430
1468
|
Parameters
|
1431
1469
|
----------
|
1432
|
-
|
1433
|
-
|
1434
|
-
|
1435
|
-
|
1436
|
-
|
1437
|
-
|
1438
|
-
|
1439
|
-
|
1470
|
+
packages : Dict[str, str], default: {}
|
1471
|
+
Packages to use for this flow. The key is the name of the package
|
1472
|
+
and the value is the version to use.
|
1473
|
+
python : str, optional, default: None
|
1474
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1475
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1440
1476
|
"""
|
1441
1477
|
...
|
1442
1478
|
|
1443
1479
|
@typing.overload
|
1444
|
-
def
|
1480
|
+
def pypi_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1445
1481
|
...
|
1446
1482
|
|
1447
|
-
def
|
1483
|
+
def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1448
1484
|
"""
|
1449
|
-
Specifies the
|
1450
|
-
|
1451
|
-
```
|
1452
|
-
@trigger_on_finish(flow='FooFlow')
|
1453
|
-
```
|
1454
|
-
or
|
1455
|
-
```
|
1456
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1457
|
-
```
|
1458
|
-
This decorator respects the @project decorator and triggers the flow
|
1459
|
-
when upstream runs within the same namespace complete successfully
|
1460
|
-
|
1461
|
-
Additionally, you can specify project aware upstream flow dependencies
|
1462
|
-
by specifying the fully qualified project_flow_name.
|
1463
|
-
```
|
1464
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1465
|
-
```
|
1466
|
-
or
|
1467
|
-
```
|
1468
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1469
|
-
```
|
1470
|
-
|
1471
|
-
You can also specify just the project or project branch (other values will be
|
1472
|
-
inferred from the current project or project branch):
|
1473
|
-
```
|
1474
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1475
|
-
```
|
1476
|
-
|
1477
|
-
Note that `branch` is typically one of:
|
1478
|
-
- `prod`
|
1479
|
-
- `user.bob`
|
1480
|
-
- `test.my_experiment`
|
1481
|
-
- `prod.staging`
|
1485
|
+
Specifies the PyPI packages for all steps of the flow.
|
1482
1486
|
|
1487
|
+
Use `@pypi_base` to set common packages required by all
|
1488
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1483
1489
|
Parameters
|
1484
1490
|
----------
|
1485
|
-
|
1486
|
-
|
1487
|
-
|
1488
|
-
|
1489
|
-
|
1490
|
-
|
1491
|
-
|
1492
|
-
|
1491
|
+
packages : Dict[str, str], default: {}
|
1492
|
+
Packages to use for this flow. The key is the name of the package
|
1493
|
+
and the value is the version to use.
|
1494
|
+
python : str, optional, default: None
|
1495
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1496
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1493
1497
|
"""
|
1494
1498
|
...
|
1495
1499
|
|
@@ -1536,90 +1540,188 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1536
1540
|
...
|
1537
1541
|
|
1538
1542
|
@typing.overload
|
1539
|
-
def
|
1543
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1540
1544
|
"""
|
1541
|
-
Specifies the
|
1545
|
+
Specifies the times when the flow should be run when running on a
|
1546
|
+
production scheduler.
|
1542
1547
|
|
1543
|
-
Use `@pypi_base` to set common packages required by all
|
1544
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1545
1548
|
Parameters
|
1546
1549
|
----------
|
1547
|
-
|
1548
|
-
|
1549
|
-
|
1550
|
-
|
1551
|
-
|
1552
|
-
|
1550
|
+
hourly : bool, default False
|
1551
|
+
Run the workflow hourly.
|
1552
|
+
daily : bool, default True
|
1553
|
+
Run the workflow daily.
|
1554
|
+
weekly : bool, default False
|
1555
|
+
Run the workflow weekly.
|
1556
|
+
cron : str, optional, default None
|
1557
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1558
|
+
specified by this expression.
|
1559
|
+
timezone : str, optional, default None
|
1560
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1561
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1553
1562
|
"""
|
1554
1563
|
...
|
1555
1564
|
|
1556
1565
|
@typing.overload
|
1557
|
-
def
|
1566
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1558
1567
|
...
|
1559
1568
|
|
1560
|
-
def
|
1569
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1561
1570
|
"""
|
1562
|
-
Specifies the
|
1571
|
+
Specifies the times when the flow should be run when running on a
|
1572
|
+
production scheduler.
|
1563
1573
|
|
1564
|
-
Use `@pypi_base` to set common packages required by all
|
1565
|
-
steps and use `@pypi` to specify step-specific overrides.
|
1566
1574
|
Parameters
|
1567
1575
|
----------
|
1568
|
-
|
1569
|
-
|
1570
|
-
|
1571
|
-
|
1572
|
-
|
1573
|
-
|
1576
|
+
hourly : bool, default False
|
1577
|
+
Run the workflow hourly.
|
1578
|
+
daily : bool, default True
|
1579
|
+
Run the workflow daily.
|
1580
|
+
weekly : bool, default False
|
1581
|
+
Run the workflow weekly.
|
1582
|
+
cron : str, optional, default None
|
1583
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1584
|
+
specified by this expression.
|
1585
|
+
timezone : str, optional, default None
|
1586
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1587
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1574
1588
|
"""
|
1575
1589
|
...
|
1576
1590
|
|
1577
|
-
|
1591
|
+
@typing.overload
|
1592
|
+
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1578
1593
|
"""
|
1579
|
-
|
1594
|
+
Specifies the event(s) that this flow depends on.
|
1580
1595
|
|
1581
|
-
|
1582
|
-
|
1583
|
-
|
1584
|
-
|
1585
|
-
|
1586
|
-
)
|
1596
|
+
```
|
1597
|
+
@trigger(event='foo')
|
1598
|
+
```
|
1599
|
+
or
|
1600
|
+
```
|
1601
|
+
@trigger(events=['foo', 'bar'])
|
1602
|
+
```
|
1587
1603
|
|
1588
|
-
|
1589
|
-
|
1590
|
-
|
1591
|
-
|
1604
|
+
Additionally, you can specify the parameter mappings
|
1605
|
+
to map event payload to Metaflow parameters for the flow.
|
1606
|
+
```
|
1607
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1608
|
+
```
|
1609
|
+
or
|
1610
|
+
```
|
1611
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1612
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1613
|
+
```
|
1592
1614
|
|
1593
|
-
|
1594
|
-
|
1595
|
-
|
1596
|
-
|
1597
|
-
|
1615
|
+
'parameters' can also be a list of strings and tuples like so:
|
1616
|
+
```
|
1617
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1618
|
+
```
|
1619
|
+
This is equivalent to:
|
1620
|
+
```
|
1621
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1622
|
+
```
|
1623
|
+
|
1624
|
+
Parameters
|
1625
|
+
----------
|
1626
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1627
|
+
Event dependency for this flow.
|
1628
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1629
|
+
Events dependency for this flow.
|
1630
|
+
options : Dict[str, Any], default {}
|
1631
|
+
Backend-specific configuration for tuning eventing behavior.
|
1632
|
+
|
1633
|
+
|
1634
|
+
"""
|
1635
|
+
...
|
1636
|
+
|
1637
|
+
@typing.overload
|
1638
|
+
def trigger(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1639
|
+
...
|
1640
|
+
|
1641
|
+
def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1642
|
+
"""
|
1643
|
+
Specifies the event(s) that this flow depends on.
|
1644
|
+
|
1645
|
+
```
|
1646
|
+
@trigger(event='foo')
|
1647
|
+
```
|
1648
|
+
or
|
1649
|
+
```
|
1650
|
+
@trigger(events=['foo', 'bar'])
|
1651
|
+
```
|
1652
|
+
|
1653
|
+
Additionally, you can specify the parameter mappings
|
1654
|
+
to map event payload to Metaflow parameters for the flow.
|
1655
|
+
```
|
1656
|
+
@trigger(event={'name':'foo', 'parameters':{'flow_param': 'event_field'}})
|
1657
|
+
```
|
1658
|
+
or
|
1659
|
+
```
|
1660
|
+
@trigger(events=[{'name':'foo', 'parameters':{'flow_param_1': 'event_field_1'},
|
1661
|
+
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1662
|
+
```
|
1663
|
+
|
1664
|
+
'parameters' can also be a list of strings and tuples like so:
|
1665
|
+
```
|
1666
|
+
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1667
|
+
```
|
1668
|
+
This is equivalent to:
|
1669
|
+
```
|
1670
|
+
@trigger(event={'name':'foo', 'parameters':{'common_name': 'common_name', 'flow_param': 'event_field'}})
|
1671
|
+
```
|
1598
1672
|
|
1599
1673
|
Parameters
|
1600
1674
|
----------
|
1601
|
-
|
1602
|
-
|
1603
|
-
|
1604
|
-
|
1675
|
+
event : Union[str, Dict[str, Any]], optional, default None
|
1676
|
+
Event dependency for this flow.
|
1677
|
+
events : List[Union[str, Dict[str, Any]]], default []
|
1678
|
+
Events dependency for this flow.
|
1679
|
+
options : Dict[str, Any], default {}
|
1680
|
+
Backend-specific configuration for tuning eventing behavior.
|
1681
|
+
|
1682
|
+
|
1605
1683
|
"""
|
1606
1684
|
...
|
1607
1685
|
|
1608
|
-
def
|
1686
|
+
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1609
1687
|
"""
|
1610
|
-
|
1611
|
-
|
1612
|
-
A project-specific namespace is created for all flows that
|
1613
|
-
use the same `@project(name)`.
|
1688
|
+
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1689
|
+
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1614
1690
|
|
1615
1691
|
Parameters
|
1616
1692
|
----------
|
1693
|
+
timeout : int
|
1694
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1695
|
+
poke_interval : int
|
1696
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1697
|
+
mode : str
|
1698
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1699
|
+
exponential_backoff : bool
|
1700
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1701
|
+
pool : str
|
1702
|
+
the slot pool this task should run in,
|
1703
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1704
|
+
soft_fail : bool
|
1705
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1617
1706
|
name : str
|
1618
|
-
|
1619
|
-
|
1620
|
-
|
1621
|
-
|
1622
|
-
|
1707
|
+
Name of the sensor on Airflow
|
1708
|
+
description : str
|
1709
|
+
Description of sensor in the Airflow UI
|
1710
|
+
external_dag_id : str
|
1711
|
+
The dag_id that contains the task you want to wait for.
|
1712
|
+
external_task_ids : List[str]
|
1713
|
+
The list of task_ids that you want to wait for.
|
1714
|
+
If None (default value) the sensor waits for the DAG. (Default: None)
|
1715
|
+
allowed_states : List[str]
|
1716
|
+
Iterable of allowed states, (Default: ['success'])
|
1717
|
+
failed_states : List[str]
|
1718
|
+
Iterable of failed or dis-allowed states. (Default: None)
|
1719
|
+
execution_delta : datetime.timedelta
|
1720
|
+
time difference with the previous execution to look at,
|
1721
|
+
the default is the same logical date as the current task or DAG. (Default: None)
|
1722
|
+
check_existence: bool
|
1723
|
+
Set to True to check if the external task exists or check if
|
1724
|
+
the DAG to wait for exists. (Default: True)
|
1623
1725
|
"""
|
1624
1726
|
...
|
1625
1727
|
|
@@ -1672,87 +1774,49 @@ def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packa
|
|
1672
1774
|
"""
|
1673
1775
|
...
|
1674
1776
|
|
1675
|
-
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1676
|
-
"""
|
1677
|
-
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
1678
|
-
This decorator only works when a flow is scheduled on Airflow and is compiled using `airflow create`. More than one `@airflow_external_task_sensor` can be added as a flow decorators. Adding more than one decorator will ensure that `start` step starts only after all sensors finish.
|
1679
|
-
|
1680
|
-
Parameters
|
1681
|
-
----------
|
1682
|
-
timeout : int
|
1683
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1684
|
-
poke_interval : int
|
1685
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1686
|
-
mode : str
|
1687
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1688
|
-
exponential_backoff : bool
|
1689
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1690
|
-
pool : str
|
1691
|
-
the slot pool this task should run in,
|
1692
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1693
|
-
soft_fail : bool
|
1694
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1695
|
-
name : str
|
1696
|
-
Name of the sensor on Airflow
|
1697
|
-
description : str
|
1698
|
-
Description of sensor in the Airflow UI
|
1699
|
-
external_dag_id : str
|
1700
|
-
The dag_id that contains the task you want to wait for.
|
1701
|
-
external_task_ids : List[str]
|
1702
|
-
The list of task_ids that you want to wait for.
|
1703
|
-
If None (default value) the sensor waits for the DAG. (Default: None)
|
1704
|
-
allowed_states : List[str]
|
1705
|
-
Iterable of allowed states, (Default: ['success'])
|
1706
|
-
failed_states : List[str]
|
1707
|
-
Iterable of failed or dis-allowed states. (Default: None)
|
1708
|
-
execution_delta : datetime.timedelta
|
1709
|
-
time difference with the previous execution to look at,
|
1710
|
-
the default is the same logical date as the current task or DAG. (Default: None)
|
1711
|
-
check_existence: bool
|
1712
|
-
Set to True to check if the external task exists or check if
|
1713
|
-
the DAG to wait for exists. (Default: True)
|
1714
|
-
"""
|
1715
|
-
...
|
1716
|
-
|
1717
1777
|
@typing.overload
|
1718
|
-
def
|
1778
|
+
def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1719
1779
|
"""
|
1720
|
-
Specifies the
|
1780
|
+
Specifies the flow(s) that this flow depends on.
|
1721
1781
|
|
1722
1782
|
```
|
1723
|
-
@
|
1783
|
+
@trigger_on_finish(flow='FooFlow')
|
1724
1784
|
```
|
1725
1785
|
or
|
1726
1786
|
```
|
1727
|
-
@
|
1787
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1728
1788
|
```
|
1789
|
+
This decorator respects the @project decorator and triggers the flow
|
1790
|
+
when upstream runs within the same namespace complete successfully
|
1729
1791
|
|
1730
|
-
Additionally, you can specify
|
1731
|
-
|
1792
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1793
|
+
by specifying the fully qualified project_flow_name.
|
1732
1794
|
```
|
1733
|
-
@
|
1795
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1734
1796
|
```
|
1735
1797
|
or
|
1736
1798
|
```
|
1737
|
-
@
|
1738
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1799
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1739
1800
|
```
|
1740
1801
|
|
1741
|
-
|
1742
|
-
|
1743
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1744
|
-
```
|
1745
|
-
This is equivalent to:
|
1802
|
+
You can also specify just the project or project branch (other values will be
|
1803
|
+
inferred from the current project or project branch):
|
1746
1804
|
```
|
1747
|
-
@
|
1805
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1748
1806
|
```
|
1749
1807
|
|
1808
|
+
Note that `branch` is typically one of:
|
1809
|
+
- `prod`
|
1810
|
+
- `user.bob`
|
1811
|
+
- `test.my_experiment`
|
1812
|
+
- `prod.staging`
|
1813
|
+
|
1750
1814
|
Parameters
|
1751
1815
|
----------
|
1752
|
-
|
1753
|
-
|
1754
|
-
|
1755
|
-
|
1816
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1817
|
+
Upstream flow dependency for this flow.
|
1818
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1819
|
+
Upstream flow dependencies for this flow.
|
1756
1820
|
options : Dict[str, Any], default {}
|
1757
1821
|
Backend-specific configuration for tuning eventing behavior.
|
1758
1822
|
|
@@ -1761,47 +1825,51 @@ def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = No
|
|
1761
1825
|
...
|
1762
1826
|
|
1763
1827
|
@typing.overload
|
1764
|
-
def
|
1828
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1765
1829
|
...
|
1766
1830
|
|
1767
|
-
def
|
1831
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1768
1832
|
"""
|
1769
|
-
Specifies the
|
1833
|
+
Specifies the flow(s) that this flow depends on.
|
1770
1834
|
|
1771
1835
|
```
|
1772
|
-
@
|
1836
|
+
@trigger_on_finish(flow='FooFlow')
|
1773
1837
|
```
|
1774
1838
|
or
|
1775
1839
|
```
|
1776
|
-
@
|
1840
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1777
1841
|
```
|
1842
|
+
This decorator respects the @project decorator and triggers the flow
|
1843
|
+
when upstream runs within the same namespace complete successfully
|
1778
1844
|
|
1779
|
-
Additionally, you can specify
|
1780
|
-
|
1845
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1846
|
+
by specifying the fully qualified project_flow_name.
|
1781
1847
|
```
|
1782
|
-
@
|
1848
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1783
1849
|
```
|
1784
1850
|
or
|
1785
1851
|
```
|
1786
|
-
@
|
1787
|
-
{'name':'bar', 'parameters':{'flow_param_2': 'event_field_2'}])
|
1852
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1788
1853
|
```
|
1789
1854
|
|
1790
|
-
|
1791
|
-
|
1792
|
-
@trigger(event={'name':'foo', 'parameters':['common_name', ('flow_param', 'event_field')]})
|
1793
|
-
```
|
1794
|
-
This is equivalent to:
|
1855
|
+
You can also specify just the project or project branch (other values will be
|
1856
|
+
inferred from the current project or project branch):
|
1795
1857
|
```
|
1796
|
-
@
|
1858
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1797
1859
|
```
|
1798
1860
|
|
1861
|
+
Note that `branch` is typically one of:
|
1862
|
+
- `prod`
|
1863
|
+
- `user.bob`
|
1864
|
+
- `test.my_experiment`
|
1865
|
+
- `prod.staging`
|
1866
|
+
|
1799
1867
|
Parameters
|
1800
1868
|
----------
|
1801
|
-
|
1802
|
-
|
1803
|
-
|
1804
|
-
|
1869
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1870
|
+
Upstream flow dependency for this flow.
|
1871
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1872
|
+
Upstream flow dependencies for this flow.
|
1805
1873
|
options : Dict[str, Any], default {}
|
1806
1874
|
Backend-specific configuration for tuning eventing behavior.
|
1807
1875
|
|
@@ -1809,55 +1877,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1809
1877
|
"""
|
1810
1878
|
...
|
1811
1879
|
|
1812
|
-
@typing.overload
|
1813
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1814
|
-
"""
|
1815
|
-
Specifies the times when the flow should be run when running on a
|
1816
|
-
production scheduler.
|
1817
|
-
|
1818
|
-
Parameters
|
1819
|
-
----------
|
1820
|
-
hourly : bool, default False
|
1821
|
-
Run the workflow hourly.
|
1822
|
-
daily : bool, default True
|
1823
|
-
Run the workflow daily.
|
1824
|
-
weekly : bool, default False
|
1825
|
-
Run the workflow weekly.
|
1826
|
-
cron : str, optional, default None
|
1827
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1828
|
-
specified by this expression.
|
1829
|
-
timezone : str, optional, default None
|
1830
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1831
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1832
|
-
"""
|
1833
|
-
...
|
1834
|
-
|
1835
|
-
@typing.overload
|
1836
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1837
|
-
...
|
1838
|
-
|
1839
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1840
|
-
"""
|
1841
|
-
Specifies the times when the flow should be run when running on a
|
1842
|
-
production scheduler.
|
1843
|
-
|
1844
|
-
Parameters
|
1845
|
-
----------
|
1846
|
-
hourly : bool, default False
|
1847
|
-
Run the workflow hourly.
|
1848
|
-
daily : bool, default True
|
1849
|
-
Run the workflow daily.
|
1850
|
-
weekly : bool, default False
|
1851
|
-
Run the workflow weekly.
|
1852
|
-
cron : str, optional, default None
|
1853
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1854
|
-
specified by this expression.
|
1855
|
-
timezone : str, optional, default None
|
1856
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1857
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1858
|
-
"""
|
1859
|
-
...
|
1860
|
-
|
1861
1880
|
def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
1862
1881
|
"""
|
1863
1882
|
Switch namespace to the one provided.
|
@@ -3006,6 +3025,36 @@ class NBRunner(object, metaclass=type):
|
|
3006
3025
|
...
|
3007
3026
|
...
|
3008
3027
|
|
3028
|
+
class Deployer(object, metaclass=type):
|
3029
|
+
def __init__(self, flow_file: str, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, cwd: typing.Optional[str] = None, **kwargs):
|
3030
|
+
...
|
3031
|
+
def _Deployer__make_function(self, deployer_class):
|
3032
|
+
"""
|
3033
|
+
Create a function for the given deployer class.
|
3034
|
+
|
3035
|
+
Parameters
|
3036
|
+
----------
|
3037
|
+
deployer_class : Type[DeployerImpl]
|
3038
|
+
Deployer implementation class.
|
3039
|
+
|
3040
|
+
Returns
|
3041
|
+
-------
|
3042
|
+
Callable
|
3043
|
+
Function that initializes and returns an instance of the deployer class.
|
3044
|
+
"""
|
3045
|
+
...
|
3046
|
+
...
|
3047
|
+
|
3048
|
+
class NBDeployer(object, metaclass=type):
|
3049
|
+
def __init__(self, flow, show_output: bool = True, profile: typing.Optional[str] = None, env: typing.Optional[typing.Dict] = None, base_dir: str = "/tmp", **kwargs):
|
3050
|
+
...
|
3051
|
+
def cleanup(self):
|
3052
|
+
"""
|
3053
|
+
Delete any temporary files created during execution.
|
3054
|
+
"""
|
3055
|
+
...
|
3056
|
+
...
|
3057
|
+
|
3009
3058
|
def get_aws_client(module, with_error = False, role_arn = None, session_vars = None, client_params = None):
|
3010
3059
|
...
|
3011
3060
|
|