ob-metaflow-stubs 5.4__py2.py3-none-any.whl → 5.5__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +510 -510
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +3 -3
- metaflow-stubs/client/__init__.pyi +5 -5
- metaflow-stubs/client/core.pyi +6 -6
- metaflow-stubs/client/filecache.pyi +3 -3
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +3 -3
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +4 -4
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +4 -4
- metaflow-stubs/metadata/metadata.pyi +3 -3
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +34 -34
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +3 -3
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +3 -3
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +5 -5
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +6 -6
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +5 -5
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +3 -3
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +3 -3
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +3 -3
- metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +4 -4
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +4 -4
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +2 -2
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +2 -2
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +4 -4
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +3 -3
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +3 -3
- metaflow-stubs/runner/metaflow_runner.pyi +3 -3
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +3 -3
- metaflow-stubs/system/system_logger.pyi +2 -2
- metaflow-stubs/system/system_monitor.pyi +3 -3
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-5.4.dist-info → ob_metaflow_stubs-5.5.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-5.5.dist-info/RECORD +138 -0
- ob_metaflow_stubs-5.4.dist-info/RECORD +0 -138
- {ob_metaflow_stubs-5.4.dist-info → ob_metaflow_stubs-5.5.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-5.4.dist-info → ob_metaflow_stubs-5.5.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,24 +1,24 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-08-
|
3
|
+
# MF version: 2.12.17.1+ob(v1) #
|
4
|
+
# Generated on 2024-08-27T02:02:03.052211 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
+
import metaflow.datastore.inputs
|
11
12
|
import io
|
12
|
-
import metaflow.client.core
|
13
|
-
import metaflow._vendor.click.types
|
14
|
-
import metaflow.parameters
|
15
|
-
import metaflow.plugins.datatools.s3.s3
|
16
13
|
import metaflow.events
|
17
|
-
import
|
14
|
+
import metaflow.plugins.datatools.s3.s3
|
18
15
|
import metaflow.metaflow_current
|
19
16
|
import metaflow.runner.metaflow_runner
|
20
|
-
import metaflow.datastore.inputs
|
21
17
|
import metaflow.flowspec
|
18
|
+
import datetime
|
19
|
+
import metaflow._vendor.click.types
|
20
|
+
import metaflow.parameters
|
21
|
+
import metaflow.client.core
|
22
22
|
import typing
|
23
23
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
24
24
|
StepFlag = typing.NewType("StepFlag", bool)
|
@@ -728,33 +728,180 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
728
728
|
...
|
729
729
|
|
730
730
|
@typing.overload
|
731
|
-
def
|
731
|
+
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
732
732
|
"""
|
733
|
-
Specifies
|
733
|
+
Specifies the resources needed when executing this step.
|
734
|
+
|
735
|
+
Use `@resources` to specify the resource requirements
|
736
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
737
|
+
|
738
|
+
You can choose the compute layer on the command line by executing e.g.
|
739
|
+
```
|
740
|
+
python myflow.py run --with batch
|
741
|
+
```
|
742
|
+
or
|
743
|
+
```
|
744
|
+
python myflow.py run --with kubernetes
|
745
|
+
```
|
746
|
+
which executes the flow on the desired system using the
|
747
|
+
requirements specified in `@resources`.
|
734
748
|
|
735
749
|
Parameters
|
736
750
|
----------
|
737
|
-
|
738
|
-
|
751
|
+
cpu : int, default 1
|
752
|
+
Number of CPUs required for this step.
|
753
|
+
gpu : int, default 0
|
754
|
+
Number of GPUs required for this step.
|
755
|
+
disk : int, optional, default None
|
756
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
757
|
+
memory : int, default 4096
|
758
|
+
Memory size (in MB) required for this step.
|
759
|
+
shared_memory : int, optional, default None
|
760
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
761
|
+
This parameter maps to the `--shm-size` option in Docker.
|
739
762
|
"""
|
740
763
|
...
|
741
764
|
|
742
765
|
@typing.overload
|
743
|
-
def
|
766
|
+
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
744
767
|
...
|
745
768
|
|
746
769
|
@typing.overload
|
747
|
-
def
|
770
|
+
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
748
771
|
...
|
749
772
|
|
750
|
-
def
|
773
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
751
774
|
"""
|
752
|
-
Specifies
|
775
|
+
Specifies the resources needed when executing this step.
|
776
|
+
|
777
|
+
Use `@resources` to specify the resource requirements
|
778
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
779
|
+
|
780
|
+
You can choose the compute layer on the command line by executing e.g.
|
781
|
+
```
|
782
|
+
python myflow.py run --with batch
|
783
|
+
```
|
784
|
+
or
|
785
|
+
```
|
786
|
+
python myflow.py run --with kubernetes
|
787
|
+
```
|
788
|
+
which executes the flow on the desired system using the
|
789
|
+
requirements specified in `@resources`.
|
753
790
|
|
754
791
|
Parameters
|
755
792
|
----------
|
756
|
-
|
757
|
-
|
793
|
+
cpu : int, default 1
|
794
|
+
Number of CPUs required for this step.
|
795
|
+
gpu : int, default 0
|
796
|
+
Number of GPUs required for this step.
|
797
|
+
disk : int, optional, default None
|
798
|
+
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
799
|
+
memory : int, default 4096
|
800
|
+
Memory size (in MB) required for this step.
|
801
|
+
shared_memory : int, optional, default None
|
802
|
+
The value for the size (in MiB) of the /dev/shm volume for this step.
|
803
|
+
This parameter maps to the `--shm-size` option in Docker.
|
804
|
+
"""
|
805
|
+
...
|
806
|
+
|
807
|
+
@typing.overload
|
808
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
809
|
+
"""
|
810
|
+
Internal decorator to support Fast bakery
|
811
|
+
"""
|
812
|
+
...
|
813
|
+
|
814
|
+
@typing.overload
|
815
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
816
|
+
...
|
817
|
+
|
818
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
819
|
+
"""
|
820
|
+
Internal decorator to support Fast bakery
|
821
|
+
"""
|
822
|
+
...
|
823
|
+
|
824
|
+
@typing.overload
|
825
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
826
|
+
"""
|
827
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
828
|
+
|
829
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
830
|
+
|
831
|
+
Parameters
|
832
|
+
----------
|
833
|
+
type : str, default 'default'
|
834
|
+
Card type.
|
835
|
+
id : str, optional, default None
|
836
|
+
If multiple cards are present, use this id to identify this card.
|
837
|
+
options : Dict[str, Any], default {}
|
838
|
+
Options passed to the card. The contents depend on the card type.
|
839
|
+
timeout : int, default 45
|
840
|
+
Interrupt reporting if it takes more than this many seconds.
|
841
|
+
|
842
|
+
|
843
|
+
"""
|
844
|
+
...
|
845
|
+
|
846
|
+
@typing.overload
|
847
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
848
|
+
...
|
849
|
+
|
850
|
+
@typing.overload
|
851
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
852
|
+
...
|
853
|
+
|
854
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
855
|
+
"""
|
856
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
857
|
+
|
858
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
859
|
+
|
860
|
+
Parameters
|
861
|
+
----------
|
862
|
+
type : str, default 'default'
|
863
|
+
Card type.
|
864
|
+
id : str, optional, default None
|
865
|
+
If multiple cards are present, use this id to identify this card.
|
866
|
+
options : Dict[str, Any], default {}
|
867
|
+
Options passed to the card. The contents depend on the card type.
|
868
|
+
timeout : int, default 45
|
869
|
+
Interrupt reporting if it takes more than this many seconds.
|
870
|
+
|
871
|
+
|
872
|
+
"""
|
873
|
+
...
|
874
|
+
|
875
|
+
@typing.overload
|
876
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
877
|
+
"""
|
878
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
879
|
+
the execution of a step.
|
880
|
+
|
881
|
+
Parameters
|
882
|
+
----------
|
883
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
884
|
+
List of secret specs, defining how the secrets are to be retrieved
|
885
|
+
"""
|
886
|
+
...
|
887
|
+
|
888
|
+
@typing.overload
|
889
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
890
|
+
...
|
891
|
+
|
892
|
+
@typing.overload
|
893
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
894
|
+
...
|
895
|
+
|
896
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
897
|
+
"""
|
898
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
899
|
+
the execution of a step.
|
900
|
+
|
901
|
+
Parameters
|
902
|
+
----------
|
903
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
904
|
+
List of secret specs, defining how the secrets are to be retrieved
|
758
905
|
"""
|
759
906
|
...
|
760
907
|
|
@@ -807,55 +954,6 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
807
954
|
"""
|
808
955
|
...
|
809
956
|
|
810
|
-
@typing.overload
|
811
|
-
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
812
|
-
"""
|
813
|
-
Specifies the PyPI packages for the step.
|
814
|
-
|
815
|
-
Information in this decorator will augment any
|
816
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
817
|
-
you can use `@pypi_base` to set packages required by all
|
818
|
-
steps and use `@pypi` to specify step-specific overrides.
|
819
|
-
|
820
|
-
Parameters
|
821
|
-
----------
|
822
|
-
packages : Dict[str, str], default: {}
|
823
|
-
Packages to use for this step. The key is the name of the package
|
824
|
-
and the value is the version to use.
|
825
|
-
python : str, optional, default: None
|
826
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
827
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
828
|
-
"""
|
829
|
-
...
|
830
|
-
|
831
|
-
@typing.overload
|
832
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
833
|
-
...
|
834
|
-
|
835
|
-
@typing.overload
|
836
|
-
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
837
|
-
...
|
838
|
-
|
839
|
-
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
840
|
-
"""
|
841
|
-
Specifies the PyPI packages for the step.
|
842
|
-
|
843
|
-
Information in this decorator will augment any
|
844
|
-
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
845
|
-
you can use `@pypi_base` to set packages required by all
|
846
|
-
steps and use `@pypi` to specify step-specific overrides.
|
847
|
-
|
848
|
-
Parameters
|
849
|
-
----------
|
850
|
-
packages : Dict[str, str], default: {}
|
851
|
-
Packages to use for this step. The key is the name of the package
|
852
|
-
and the value is the version to use.
|
853
|
-
python : str, optional, default: None
|
854
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
855
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
856
|
-
"""
|
857
|
-
...
|
858
|
-
|
859
957
|
@typing.overload
|
860
958
|
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
861
959
|
"""
|
@@ -914,35 +1012,59 @@ def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
914
1012
|
...
|
915
1013
|
|
916
1014
|
@typing.overload
|
917
|
-
def
|
1015
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
918
1016
|
"""
|
919
|
-
Specifies
|
920
|
-
|
1017
|
+
Specifies the Conda environment for the step.
|
1018
|
+
|
1019
|
+
Information in this decorator will augment any
|
1020
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1021
|
+
you can use `@conda_base` to set packages required by all
|
1022
|
+
steps and use `@conda` to specify step-specific overrides.
|
921
1023
|
|
922
1024
|
Parameters
|
923
1025
|
----------
|
924
|
-
|
925
|
-
|
1026
|
+
packages : Dict[str, str], default {}
|
1027
|
+
Packages to use for this step. The key is the name of the package
|
1028
|
+
and the value is the version to use.
|
1029
|
+
libraries : Dict[str, str], default {}
|
1030
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1031
|
+
python : str, optional, default None
|
1032
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1033
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1034
|
+
disabled : bool, default False
|
1035
|
+
If set to True, disables @conda.
|
926
1036
|
"""
|
927
1037
|
...
|
928
1038
|
|
929
1039
|
@typing.overload
|
930
|
-
def
|
1040
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
931
1041
|
...
|
932
1042
|
|
933
1043
|
@typing.overload
|
934
|
-
def
|
1044
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
935
1045
|
...
|
936
1046
|
|
937
|
-
def
|
1047
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
938
1048
|
"""
|
939
|
-
Specifies
|
940
|
-
|
1049
|
+
Specifies the Conda environment for the step.
|
1050
|
+
|
1051
|
+
Information in this decorator will augment any
|
1052
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1053
|
+
you can use `@conda_base` to set packages required by all
|
1054
|
+
steps and use `@conda` to specify step-specific overrides.
|
941
1055
|
|
942
1056
|
Parameters
|
943
1057
|
----------
|
944
|
-
|
945
|
-
|
1058
|
+
packages : Dict[str, str], default {}
|
1059
|
+
Packages to use for this step. The key is the name of the package
|
1060
|
+
and the value is the version to use.
|
1061
|
+
libraries : Dict[str, str], default {}
|
1062
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1063
|
+
python : str, optional, default None
|
1064
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1065
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1066
|
+
disabled : bool, default False
|
1067
|
+
If set to True, disables @conda.
|
946
1068
|
"""
|
947
1069
|
...
|
948
1070
|
|
@@ -999,176 +1121,6 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
999
1121
|
"""
|
1000
1122
|
...
|
1001
1123
|
|
1002
|
-
@typing.overload
|
1003
|
-
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1004
|
-
"""
|
1005
|
-
Specifies the Conda environment for the step.
|
1006
|
-
|
1007
|
-
Information in this decorator will augment any
|
1008
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1009
|
-
you can use `@conda_base` to set packages required by all
|
1010
|
-
steps and use `@conda` to specify step-specific overrides.
|
1011
|
-
|
1012
|
-
Parameters
|
1013
|
-
----------
|
1014
|
-
packages : Dict[str, str], default {}
|
1015
|
-
Packages to use for this step. The key is the name of the package
|
1016
|
-
and the value is the version to use.
|
1017
|
-
libraries : Dict[str, str], default {}
|
1018
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1019
|
-
python : str, optional, default None
|
1020
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1021
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1022
|
-
disabled : bool, default False
|
1023
|
-
If set to True, disables @conda.
|
1024
|
-
"""
|
1025
|
-
...
|
1026
|
-
|
1027
|
-
@typing.overload
|
1028
|
-
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1029
|
-
...
|
1030
|
-
|
1031
|
-
@typing.overload
|
1032
|
-
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1033
|
-
...
|
1034
|
-
|
1035
|
-
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1036
|
-
"""
|
1037
|
-
Specifies the Conda environment for the step.
|
1038
|
-
|
1039
|
-
Information in this decorator will augment any
|
1040
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
1041
|
-
you can use `@conda_base` to set packages required by all
|
1042
|
-
steps and use `@conda` to specify step-specific overrides.
|
1043
|
-
|
1044
|
-
Parameters
|
1045
|
-
----------
|
1046
|
-
packages : Dict[str, str], default {}
|
1047
|
-
Packages to use for this step. The key is the name of the package
|
1048
|
-
and the value is the version to use.
|
1049
|
-
libraries : Dict[str, str], default {}
|
1050
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1051
|
-
python : str, optional, default None
|
1052
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1053
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1054
|
-
disabled : bool, default False
|
1055
|
-
If set to True, disables @conda.
|
1056
|
-
"""
|
1057
|
-
...
|
1058
|
-
|
1059
|
-
@typing.overload
|
1060
|
-
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1061
|
-
"""
|
1062
|
-
Specifies the resources needed when executing this step.
|
1063
|
-
|
1064
|
-
Use `@resources` to specify the resource requirements
|
1065
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1066
|
-
|
1067
|
-
You can choose the compute layer on the command line by executing e.g.
|
1068
|
-
```
|
1069
|
-
python myflow.py run --with batch
|
1070
|
-
```
|
1071
|
-
or
|
1072
|
-
```
|
1073
|
-
python myflow.py run --with kubernetes
|
1074
|
-
```
|
1075
|
-
which executes the flow on the desired system using the
|
1076
|
-
requirements specified in `@resources`.
|
1077
|
-
|
1078
|
-
Parameters
|
1079
|
-
----------
|
1080
|
-
cpu : int, default 1
|
1081
|
-
Number of CPUs required for this step.
|
1082
|
-
gpu : int, default 0
|
1083
|
-
Number of GPUs required for this step.
|
1084
|
-
disk : int, optional, default None
|
1085
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1086
|
-
memory : int, default 4096
|
1087
|
-
Memory size (in MB) required for this step.
|
1088
|
-
shared_memory : int, optional, default None
|
1089
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1090
|
-
This parameter maps to the `--shm-size` option in Docker.
|
1091
|
-
"""
|
1092
|
-
...
|
1093
|
-
|
1094
|
-
@typing.overload
|
1095
|
-
def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1096
|
-
...
|
1097
|
-
|
1098
|
-
@typing.overload
|
1099
|
-
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1100
|
-
...
|
1101
|
-
|
1102
|
-
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
1103
|
-
"""
|
1104
|
-
Specifies the resources needed when executing this step.
|
1105
|
-
|
1106
|
-
Use `@resources` to specify the resource requirements
|
1107
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
1108
|
-
|
1109
|
-
You can choose the compute layer on the command line by executing e.g.
|
1110
|
-
```
|
1111
|
-
python myflow.py run --with batch
|
1112
|
-
```
|
1113
|
-
or
|
1114
|
-
```
|
1115
|
-
python myflow.py run --with kubernetes
|
1116
|
-
```
|
1117
|
-
which executes the flow on the desired system using the
|
1118
|
-
requirements specified in `@resources`.
|
1119
|
-
|
1120
|
-
Parameters
|
1121
|
-
----------
|
1122
|
-
cpu : int, default 1
|
1123
|
-
Number of CPUs required for this step.
|
1124
|
-
gpu : int, default 0
|
1125
|
-
Number of GPUs required for this step.
|
1126
|
-
disk : int, optional, default None
|
1127
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
1128
|
-
memory : int, default 4096
|
1129
|
-
Memory size (in MB) required for this step.
|
1130
|
-
shared_memory : int, optional, default None
|
1131
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
1132
|
-
This parameter maps to the `--shm-size` option in Docker.
|
1133
|
-
"""
|
1134
|
-
...
|
1135
|
-
|
1136
|
-
@typing.overload
|
1137
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1138
|
-
"""
|
1139
|
-
Internal decorator to support Fast bakery
|
1140
|
-
"""
|
1141
|
-
...
|
1142
|
-
|
1143
|
-
@typing.overload
|
1144
|
-
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1145
|
-
...
|
1146
|
-
|
1147
|
-
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1148
|
-
"""
|
1149
|
-
Internal decorator to support Fast bakery
|
1150
|
-
"""
|
1151
|
-
...
|
1152
|
-
|
1153
|
-
@typing.overload
|
1154
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1155
|
-
"""
|
1156
|
-
Decorator prototype for all step decorators. This function gets specialized
|
1157
|
-
and imported for all decorators types by _import_plugin_decorators().
|
1158
|
-
"""
|
1159
|
-
...
|
1160
|
-
|
1161
|
-
@typing.overload
|
1162
|
-
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1163
|
-
...
|
1164
|
-
|
1165
|
-
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1166
|
-
"""
|
1167
|
-
Decorator prototype for all step decorators. This function gets specialized
|
1168
|
-
and imported for all decorators types by _import_plugin_decorators().
|
1169
|
-
"""
|
1170
|
-
...
|
1171
|
-
|
1172
1124
|
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1173
1125
|
"""
|
1174
1126
|
Specifies that this step should execute on Kubernetes.
|
@@ -1232,53 +1184,150 @@ def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: ty
|
|
1232
1184
|
...
|
1233
1185
|
|
1234
1186
|
@typing.overload
|
1235
|
-
def
|
1187
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1236
1188
|
"""
|
1237
|
-
|
1189
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1238
1190
|
|
1239
|
-
|
1191
|
+
Parameters
|
1192
|
+
----------
|
1193
|
+
vars : Dict[str, str], default {}
|
1194
|
+
Dictionary of environment variables to set.
|
1195
|
+
"""
|
1196
|
+
...
|
1197
|
+
|
1198
|
+
@typing.overload
|
1199
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1200
|
+
...
|
1201
|
+
|
1202
|
+
@typing.overload
|
1203
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1204
|
+
...
|
1205
|
+
|
1206
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1207
|
+
"""
|
1208
|
+
Specifies environment variables to be set prior to the execution of a step.
|
1240
1209
|
|
1241
1210
|
Parameters
|
1242
1211
|
----------
|
1243
|
-
|
1244
|
-
|
1245
|
-
|
1246
|
-
|
1247
|
-
|
1248
|
-
|
1249
|
-
|
1250
|
-
|
1212
|
+
vars : Dict[str, str], default {}
|
1213
|
+
Dictionary of environment variables to set.
|
1214
|
+
"""
|
1215
|
+
...
|
1216
|
+
|
1217
|
+
@typing.overload
|
1218
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1219
|
+
"""
|
1220
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1221
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1222
|
+
"""
|
1223
|
+
...
|
1224
|
+
|
1225
|
+
@typing.overload
|
1226
|
+
def parallel(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1227
|
+
...
|
1228
|
+
|
1229
|
+
def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
1230
|
+
"""
|
1231
|
+
Decorator prototype for all step decorators. This function gets specialized
|
1232
|
+
and imported for all decorators types by _import_plugin_decorators().
|
1233
|
+
"""
|
1234
|
+
...
|
1235
|
+
|
1236
|
+
@typing.overload
|
1237
|
+
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1238
|
+
"""
|
1239
|
+
Specifies the PyPI packages for the step.
|
1251
1240
|
|
1241
|
+
Information in this decorator will augment any
|
1242
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1243
|
+
you can use `@pypi_base` to set packages required by all
|
1244
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1252
1245
|
|
1246
|
+
Parameters
|
1247
|
+
----------
|
1248
|
+
packages : Dict[str, str], default: {}
|
1249
|
+
Packages to use for this step. The key is the name of the package
|
1250
|
+
and the value is the version to use.
|
1251
|
+
python : str, optional, default: None
|
1252
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1253
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1253
1254
|
"""
|
1254
1255
|
...
|
1255
1256
|
|
1256
1257
|
@typing.overload
|
1257
|
-
def
|
1258
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1258
1259
|
...
|
1259
1260
|
|
1260
1261
|
@typing.overload
|
1261
|
-
def
|
1262
|
+
def pypi(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1262
1263
|
...
|
1263
1264
|
|
1264
|
-
def
|
1265
|
+
def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None):
|
1265
1266
|
"""
|
1266
|
-
|
1267
|
+
Specifies the PyPI packages for the step.
|
1267
1268
|
|
1268
|
-
|
1269
|
+
Information in this decorator will augment any
|
1270
|
+
attributes set in the `@pyi_base` flow-level decorator. Hence,
|
1271
|
+
you can use `@pypi_base` to set packages required by all
|
1272
|
+
steps and use `@pypi` to specify step-specific overrides.
|
1269
1273
|
|
1270
1274
|
Parameters
|
1271
1275
|
----------
|
1272
|
-
|
1273
|
-
|
1274
|
-
|
1275
|
-
|
1276
|
-
|
1277
|
-
|
1278
|
-
|
1279
|
-
|
1276
|
+
packages : Dict[str, str], default: {}
|
1277
|
+
Packages to use for this step. The key is the name of the package
|
1278
|
+
and the value is the version to use.
|
1279
|
+
python : str, optional, default: None
|
1280
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1281
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1282
|
+
"""
|
1283
|
+
...
|
1284
|
+
|
1285
|
+
@typing.overload
|
1286
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1287
|
+
"""
|
1288
|
+
Specifies the times when the flow should be run when running on a
|
1289
|
+
production scheduler.
|
1280
1290
|
|
1291
|
+
Parameters
|
1292
|
+
----------
|
1293
|
+
hourly : bool, default False
|
1294
|
+
Run the workflow hourly.
|
1295
|
+
daily : bool, default True
|
1296
|
+
Run the workflow daily.
|
1297
|
+
weekly : bool, default False
|
1298
|
+
Run the workflow weekly.
|
1299
|
+
cron : str, optional, default None
|
1300
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1301
|
+
specified by this expression.
|
1302
|
+
timezone : str, optional, default None
|
1303
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1304
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1305
|
+
"""
|
1306
|
+
...
|
1307
|
+
|
1308
|
+
@typing.overload
|
1309
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1310
|
+
...
|
1311
|
+
|
1312
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1313
|
+
"""
|
1314
|
+
Specifies the times when the flow should be run when running on a
|
1315
|
+
production scheduler.
|
1281
1316
|
|
1317
|
+
Parameters
|
1318
|
+
----------
|
1319
|
+
hourly : bool, default False
|
1320
|
+
Run the workflow hourly.
|
1321
|
+
daily : bool, default True
|
1322
|
+
Run the workflow daily.
|
1323
|
+
weekly : bool, default False
|
1324
|
+
Run the workflow weekly.
|
1325
|
+
cron : str, optional, default None
|
1326
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1327
|
+
specified by this expression.
|
1328
|
+
timezone : str, optional, default None
|
1329
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1330
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1282
1331
|
"""
|
1283
1332
|
...
|
1284
1333
|
|
@@ -1321,148 +1370,143 @@ def pypi_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packag
|
|
1321
1370
|
"""
|
1322
1371
|
...
|
1323
1372
|
|
1324
|
-
|
1325
|
-
def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1373
|
+
def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, bucket_key: typing.Union[str, typing.List[str]], bucket_name: str, wildcard_match: bool, aws_conn_id: str, verify: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1326
1374
|
"""
|
1327
|
-
|
1375
|
+
The `@airflow_s3_key_sensor` decorator attaches a Airflow [S3KeySensor](https://airflow.apache.org/docs/apache-airflow-providers-amazon/stable/_api/airflow/providers/amazon/aws/sensors/s3/index.html#airflow.providers.amazon.aws.sensors.s3.S3KeySensor)
|
1376
|
+
before the start step of the flow. This decorator only works when a flow is scheduled on Airflow
|
1377
|
+
and is compiled using `airflow create`. More than one `@airflow_s3_key_sensor` can be
|
1378
|
+
added as a flow decorators. Adding more than one decorator will ensure that `start` step
|
1379
|
+
starts only after all sensors finish.
|
1328
1380
|
|
1329
|
-
|
1330
|
-
|
1331
|
-
|
1332
|
-
|
1333
|
-
|
1334
|
-
|
1335
|
-
|
1336
|
-
|
1337
|
-
|
1381
|
+
Parameters
|
1382
|
+
----------
|
1383
|
+
timeout : int
|
1384
|
+
Time, in seconds before the task times out and fails. (Default: 3600)
|
1385
|
+
poke_interval : int
|
1386
|
+
Time in seconds that the job should wait in between each try. (Default: 60)
|
1387
|
+
mode : str
|
1388
|
+
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1389
|
+
exponential_backoff : bool
|
1390
|
+
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1391
|
+
pool : str
|
1392
|
+
the slot pool this task should run in,
|
1393
|
+
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1394
|
+
soft_fail : bool
|
1395
|
+
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1396
|
+
name : str
|
1397
|
+
Name of the sensor on Airflow
|
1398
|
+
description : str
|
1399
|
+
Description of sensor in the Airflow UI
|
1400
|
+
bucket_key : Union[str, List[str]]
|
1401
|
+
The key(s) being waited on. Supports full s3:// style url or relative path from root level.
|
1402
|
+
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1403
|
+
bucket_name : str
|
1404
|
+
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1405
|
+
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1406
|
+
wildcard_match : bool
|
1407
|
+
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1408
|
+
aws_conn_id : str
|
1409
|
+
a reference to the s3 connection on Airflow. (Default: None)
|
1410
|
+
verify : bool
|
1411
|
+
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1412
|
+
"""
|
1413
|
+
...
|
1414
|
+
|
1415
|
+
def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1416
|
+
"""
|
1417
|
+
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
1338
1418
|
|
1339
|
-
|
1340
|
-
|
1341
|
-
|
1342
|
-
|
1343
|
-
|
1344
|
-
|
1345
|
-
```
|
1346
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1347
|
-
```
|
1419
|
+
User code call
|
1420
|
+
-----------
|
1421
|
+
@nim(
|
1422
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
1423
|
+
backend='managed'
|
1424
|
+
)
|
1348
1425
|
|
1349
|
-
|
1350
|
-
|
1351
|
-
|
1352
|
-
|
1353
|
-
```
|
1426
|
+
Valid backend options
|
1427
|
+
---------------------
|
1428
|
+
- 'managed': Outerbounds selects a compute provider based on the model.
|
1429
|
+
- 🚧 'dataplane': Run in your account.
|
1354
1430
|
|
1355
|
-
|
1356
|
-
|
1357
|
-
|
1358
|
-
|
1359
|
-
|
1431
|
+
Valid model options
|
1432
|
+
----------------
|
1433
|
+
- 'meta/llama3-8b-instruct': 8B parameter model
|
1434
|
+
- 'meta/llama3-70b-instruct': 70B parameter model
|
1435
|
+
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
1360
1436
|
|
1361
1437
|
Parameters
|
1362
1438
|
----------
|
1363
|
-
|
1364
|
-
|
1365
|
-
|
1366
|
-
|
1367
|
-
options : Dict[str, Any], default {}
|
1368
|
-
Backend-specific configuration for tuning eventing behavior.
|
1369
|
-
|
1370
|
-
|
1439
|
+
models: list[NIM]
|
1440
|
+
List of NIM containers running models in sidecars.
|
1441
|
+
backend: str
|
1442
|
+
Compute provider to run the NIM container.
|
1371
1443
|
"""
|
1372
1444
|
...
|
1373
1445
|
|
1374
1446
|
@typing.overload
|
1375
|
-
def
|
1376
|
-
...
|
1377
|
-
|
1378
|
-
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1447
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1379
1448
|
"""
|
1380
|
-
Specifies the
|
1381
|
-
|
1382
|
-
```
|
1383
|
-
@trigger_on_finish(flow='FooFlow')
|
1384
|
-
```
|
1385
|
-
or
|
1386
|
-
```
|
1387
|
-
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1388
|
-
```
|
1389
|
-
This decorator respects the @project decorator and triggers the flow
|
1390
|
-
when upstream runs within the same namespace complete successfully
|
1391
|
-
|
1392
|
-
Additionally, you can specify project aware upstream flow dependencies
|
1393
|
-
by specifying the fully qualified project_flow_name.
|
1394
|
-
```
|
1395
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1396
|
-
```
|
1397
|
-
or
|
1398
|
-
```
|
1399
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1400
|
-
```
|
1401
|
-
|
1402
|
-
You can also specify just the project or project branch (other values will be
|
1403
|
-
inferred from the current project or project branch):
|
1404
|
-
```
|
1405
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1406
|
-
```
|
1449
|
+
Specifies the Conda environment for all steps of the flow.
|
1407
1450
|
|
1408
|
-
|
1409
|
-
|
1410
|
-
- `user.bob`
|
1411
|
-
- `test.my_experiment`
|
1412
|
-
- `prod.staging`
|
1451
|
+
Use `@conda_base` to set common libraries required by all
|
1452
|
+
steps and use `@conda` to specify step-specific additions.
|
1413
1453
|
|
1414
1454
|
Parameters
|
1415
1455
|
----------
|
1416
|
-
|
1417
|
-
|
1418
|
-
|
1419
|
-
|
1420
|
-
|
1421
|
-
|
1456
|
+
packages : Dict[str, str], default {}
|
1457
|
+
Packages to use for this flow. The key is the name of the package
|
1458
|
+
and the value is the version to use.
|
1459
|
+
libraries : Dict[str, str], default {}
|
1460
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1461
|
+
python : str, optional, default None
|
1462
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1463
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1464
|
+
disabled : bool, default False
|
1465
|
+
If set to True, disables Conda.
|
1466
|
+
"""
|
1467
|
+
...
|
1468
|
+
|
1469
|
+
@typing.overload
|
1470
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1471
|
+
...
|
1472
|
+
|
1473
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1474
|
+
"""
|
1475
|
+
Specifies the Conda environment for all steps of the flow.
|
1422
1476
|
|
1477
|
+
Use `@conda_base` to set common libraries required by all
|
1478
|
+
steps and use `@conda` to specify step-specific additions.
|
1423
1479
|
|
1480
|
+
Parameters
|
1481
|
+
----------
|
1482
|
+
packages : Dict[str, str], default {}
|
1483
|
+
Packages to use for this flow. The key is the name of the package
|
1484
|
+
and the value is the version to use.
|
1485
|
+
libraries : Dict[str, str], default {}
|
1486
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1487
|
+
python : str, optional, default None
|
1488
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1489
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1490
|
+
disabled : bool, default False
|
1491
|
+
If set to True, disables Conda.
|
1424
1492
|
"""
|
1425
1493
|
...
|
1426
1494
|
|
1427
|
-
def
|
1495
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1428
1496
|
"""
|
1429
|
-
|
1430
|
-
|
1431
|
-
|
1432
|
-
|
1433
|
-
starts only after all sensors finish.
|
1497
|
+
Specifies what flows belong to the same project.
|
1498
|
+
|
1499
|
+
A project-specific namespace is created for all flows that
|
1500
|
+
use the same `@project(name)`.
|
1434
1501
|
|
1435
1502
|
Parameters
|
1436
1503
|
----------
|
1437
|
-
timeout : int
|
1438
|
-
Time, in seconds before the task times out and fails. (Default: 3600)
|
1439
|
-
poke_interval : int
|
1440
|
-
Time in seconds that the job should wait in between each try. (Default: 60)
|
1441
|
-
mode : str
|
1442
|
-
How the sensor operates. Options are: { poke | reschedule }. (Default: "poke")
|
1443
|
-
exponential_backoff : bool
|
1444
|
-
allow progressive longer waits between pokes by using exponential backoff algorithm. (Default: True)
|
1445
|
-
pool : str
|
1446
|
-
the slot pool this task should run in,
|
1447
|
-
slot pools are a way to limit concurrency for certain tasks. (Default:None)
|
1448
|
-
soft_fail : bool
|
1449
|
-
Set to true to mark the task as SKIPPED on failure. (Default: False)
|
1450
1504
|
name : str
|
1451
|
-
|
1452
|
-
|
1453
|
-
|
1454
|
-
|
1455
|
-
|
1456
|
-
When it's specified as a full s3:// url, please leave `bucket_name` as None
|
1457
|
-
bucket_name : str
|
1458
|
-
Name of the S3 bucket. Only needed when bucket_key is not provided as a full s3:// url.
|
1459
|
-
When specified, all the keys passed to bucket_key refers to this bucket. (Default:None)
|
1460
|
-
wildcard_match : bool
|
1461
|
-
whether the bucket_key should be interpreted as a Unix wildcard pattern. (Default: False)
|
1462
|
-
aws_conn_id : str
|
1463
|
-
a reference to the s3 connection on Airflow. (Default: None)
|
1464
|
-
verify : bool
|
1465
|
-
Whether or not to verify SSL certificates for S3 connection. (Default: None)
|
1505
|
+
Project name. Make sure that the name is unique amongst all
|
1506
|
+
projects that use the same production scheduler. The name may
|
1507
|
+
contain only lowercase alphanumeric characters and underscores.
|
1508
|
+
|
1509
|
+
|
1466
1510
|
"""
|
1467
1511
|
...
|
1468
1512
|
|
@@ -1561,24 +1605,6 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1561
1605
|
"""
|
1562
1606
|
...
|
1563
1607
|
|
1564
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1565
|
-
"""
|
1566
|
-
Specifies what flows belong to the same project.
|
1567
|
-
|
1568
|
-
A project-specific namespace is created for all flows that
|
1569
|
-
use the same `@project(name)`.
|
1570
|
-
|
1571
|
-
Parameters
|
1572
|
-
----------
|
1573
|
-
name : str
|
1574
|
-
Project name. Make sure that the name is unique amongst all
|
1575
|
-
projects that use the same production scheduler. The name may
|
1576
|
-
contain only lowercase alphanumeric characters and underscores.
|
1577
|
-
|
1578
|
-
|
1579
|
-
"""
|
1580
|
-
...
|
1581
|
-
|
1582
1608
|
def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str, exponential_backoff: bool, pool: str, soft_fail: bool, name: str, description: str, external_dag_id: str, external_task_ids: typing.List[str], allowed_states: typing.List[str], failed_states: typing.List[str], execution_delta: "datetime.timedelta", check_existence: bool) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1583
1609
|
"""
|
1584
1610
|
The `@airflow_external_task_sensor` decorator attaches a Airflow [ExternalTaskSensor](https://airflow.apache.org/docs/apache-airflow/stable/_api/airflow/sensors/external_task/index.html#airflow.sensors.external_task.ExternalTaskSensor) before the start step of the flow.
|
@@ -1621,132 +1647,106 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1621
1647
|
"""
|
1622
1648
|
...
|
1623
1649
|
|
1624
|
-
|
1650
|
+
@typing.overload
|
1651
|
+
def trigger_on_finish(*, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1625
1652
|
"""
|
1626
|
-
|
1653
|
+
Specifies the flow(s) that this flow depends on.
|
1627
1654
|
|
1628
|
-
|
1629
|
-
|
1630
|
-
|
1631
|
-
|
1632
|
-
|
1633
|
-
)
|
1655
|
+
```
|
1656
|
+
@trigger_on_finish(flow='FooFlow')
|
1657
|
+
```
|
1658
|
+
or
|
1659
|
+
```
|
1660
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1661
|
+
```
|
1662
|
+
This decorator respects the @project decorator and triggers the flow
|
1663
|
+
when upstream runs within the same namespace complete successfully
|
1634
1664
|
|
1635
|
-
|
1636
|
-
|
1637
|
-
|
1638
|
-
|
1665
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1666
|
+
by specifying the fully qualified project_flow_name.
|
1667
|
+
```
|
1668
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1669
|
+
```
|
1670
|
+
or
|
1671
|
+
```
|
1672
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1673
|
+
```
|
1639
1674
|
|
1640
|
-
|
1641
|
-
|
1642
|
-
|
1643
|
-
|
1644
|
-
|
1675
|
+
You can also specify just the project or project branch (other values will be
|
1676
|
+
inferred from the current project or project branch):
|
1677
|
+
```
|
1678
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1679
|
+
```
|
1645
1680
|
|
1646
|
-
|
1647
|
-
|
1648
|
-
|
1649
|
-
|
1650
|
-
|
1651
|
-
Compute provider to run the NIM container.
|
1652
|
-
"""
|
1653
|
-
...
|
1654
|
-
|
1655
|
-
@typing.overload
|
1656
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1657
|
-
"""
|
1658
|
-
Specifies the times when the flow should be run when running on a
|
1659
|
-
production scheduler.
|
1681
|
+
Note that `branch` is typically one of:
|
1682
|
+
- `prod`
|
1683
|
+
- `user.bob`
|
1684
|
+
- `test.my_experiment`
|
1685
|
+
- `prod.staging`
|
1660
1686
|
|
1661
1687
|
Parameters
|
1662
1688
|
----------
|
1663
|
-
|
1664
|
-
|
1665
|
-
|
1666
|
-
|
1667
|
-
|
1668
|
-
|
1669
|
-
|
1670
|
-
|
1671
|
-
specified by this expression.
|
1672
|
-
timezone : str, optional, default None
|
1673
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1674
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1689
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1690
|
+
Upstream flow dependency for this flow.
|
1691
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1692
|
+
Upstream flow dependencies for this flow.
|
1693
|
+
options : Dict[str, Any], default {}
|
1694
|
+
Backend-specific configuration for tuning eventing behavior.
|
1695
|
+
|
1696
|
+
|
1675
1697
|
"""
|
1676
1698
|
...
|
1677
1699
|
|
1678
1700
|
@typing.overload
|
1679
|
-
def
|
1701
|
+
def trigger_on_finish(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1680
1702
|
...
|
1681
1703
|
|
1682
|
-
def
|
1704
|
+
def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, flow: typing.Union[str, typing.Dict[str, str], None] = None, flows: typing.List[typing.Union[str, typing.Dict[str, str]]] = [], options: typing.Dict[str, typing.Any] = {}):
|
1683
1705
|
"""
|
1684
|
-
Specifies the
|
1685
|
-
production scheduler.
|
1706
|
+
Specifies the flow(s) that this flow depends on.
|
1686
1707
|
|
1687
|
-
|
1688
|
-
|
1689
|
-
|
1690
|
-
|
1691
|
-
|
1692
|
-
|
1693
|
-
|
1694
|
-
|
1695
|
-
|
1696
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1697
|
-
specified by this expression.
|
1698
|
-
timezone : str, optional, default None
|
1699
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1700
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1701
|
-
"""
|
1702
|
-
...
|
1703
|
-
|
1704
|
-
@typing.overload
|
1705
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1706
|
-
"""
|
1707
|
-
Specifies the Conda environment for all steps of the flow.
|
1708
|
+
```
|
1709
|
+
@trigger_on_finish(flow='FooFlow')
|
1710
|
+
```
|
1711
|
+
or
|
1712
|
+
```
|
1713
|
+
@trigger_on_finish(flows=['FooFlow', 'BarFlow'])
|
1714
|
+
```
|
1715
|
+
This decorator respects the @project decorator and triggers the flow
|
1716
|
+
when upstream runs within the same namespace complete successfully
|
1708
1717
|
|
1709
|
-
|
1710
|
-
|
1718
|
+
Additionally, you can specify project aware upstream flow dependencies
|
1719
|
+
by specifying the fully qualified project_flow_name.
|
1720
|
+
```
|
1721
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1722
|
+
```
|
1723
|
+
or
|
1724
|
+
```
|
1725
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1726
|
+
```
|
1711
1727
|
|
1712
|
-
|
1713
|
-
|
1714
|
-
|
1715
|
-
|
1716
|
-
|
1717
|
-
libraries : Dict[str, str], default {}
|
1718
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1719
|
-
python : str, optional, default None
|
1720
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1721
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1722
|
-
disabled : bool, default False
|
1723
|
-
If set to True, disables Conda.
|
1724
|
-
"""
|
1725
|
-
...
|
1726
|
-
|
1727
|
-
@typing.overload
|
1728
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1729
|
-
...
|
1730
|
-
|
1731
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1732
|
-
"""
|
1733
|
-
Specifies the Conda environment for all steps of the flow.
|
1728
|
+
You can also specify just the project or project branch (other values will be
|
1729
|
+
inferred from the current project or project branch):
|
1730
|
+
```
|
1731
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1732
|
+
```
|
1734
1733
|
|
1735
|
-
|
1736
|
-
|
1734
|
+
Note that `branch` is typically one of:
|
1735
|
+
- `prod`
|
1736
|
+
- `user.bob`
|
1737
|
+
- `test.my_experiment`
|
1738
|
+
- `prod.staging`
|
1737
1739
|
|
1738
1740
|
Parameters
|
1739
1741
|
----------
|
1740
|
-
|
1741
|
-
|
1742
|
-
|
1743
|
-
|
1744
|
-
|
1745
|
-
|
1746
|
-
|
1747
|
-
|
1748
|
-
disabled : bool, default False
|
1749
|
-
If set to True, disables Conda.
|
1742
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1743
|
+
Upstream flow dependency for this flow.
|
1744
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1745
|
+
Upstream flow dependencies for this flow.
|
1746
|
+
options : Dict[str, Any], default {}
|
1747
|
+
Backend-specific configuration for tuning eventing behavior.
|
1748
|
+
|
1749
|
+
|
1750
1750
|
"""
|
1751
1751
|
...
|
1752
1752
|
|