ob-metaflow-stubs 6.0.3.114__py2.py3-none-any.whl → 6.0.3.115__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- metaflow-stubs/__init__.pyi +498 -498
- metaflow-stubs/cards.pyi +5 -5
- metaflow-stubs/cli.pyi +3 -3
- metaflow-stubs/client/__init__.pyi +5 -5
- metaflow-stubs/client/core.pyi +11 -9
- metaflow-stubs/client/filecache.pyi +2 -2
- metaflow-stubs/clone_util.pyi +2 -2
- metaflow-stubs/events.pyi +2 -2
- metaflow-stubs/exception.pyi +2 -2
- metaflow-stubs/flowspec.pyi +5 -5
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +5 -5
- metaflow-stubs/info_file.pyi +2 -2
- metaflow-stubs/metadata/metadata.pyi +6 -3
- metaflow-stubs/metadata/util.pyi +2 -2
- metaflow-stubs/metaflow_config.pyi +2 -2
- metaflow-stubs/metaflow_current.pyi +68 -68
- metaflow-stubs/mflog/mflog.pyi +2 -2
- metaflow-stubs/multicore_utils.pyi +2 -2
- metaflow-stubs/parameters.pyi +3 -3
- metaflow-stubs/plugins/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
- metaflow-stubs/plugins/airflow/exception.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +3 -3
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +4 -4
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +4 -4
- metaflow-stubs/plugins/argo/__init__.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
- metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +5 -5
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +7 -7
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +2 -2
- metaflow-stubs/plugins/aws/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
- metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +5 -5
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +2 -2
- metaflow-stubs/plugins/azure/__init__.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
- metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/cards/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_cli.pyi +4 -4
- metaflow-stubs/plugins/cards/card_client.pyi +3 -3
- metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
- metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
- metaflow-stubs/plugins/cards/card_decorator.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
- metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +3 -3
- metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
- metaflow-stubs/plugins/cards/component_serializer.pyi +4 -4
- metaflow-stubs/plugins/cards/exception.pyi +2 -2
- metaflow-stubs/plugins/catch_decorator.pyi +4 -4
- metaflow-stubs/plugins/datatools/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/local.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +3 -3
- metaflow-stubs/plugins/datatools/s3/s3.pyi +7 -7
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
- metaflow-stubs/plugins/debug_logger.pyi +2 -2
- metaflow-stubs/plugins/debug_monitor.pyi +2 -2
- metaflow-stubs/plugins/environment_decorator.pyi +2 -2
- metaflow-stubs/plugins/events_decorator.pyi +3 -3
- metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
- metaflow-stubs/plugins/frameworks/pytorch.pyi +3 -3
- metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
- metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
- metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +3 -3
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +5 -5
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
- metaflow-stubs/plugins/logs_cli.pyi +3 -3
- metaflow-stubs/plugins/package_cli.pyi +2 -2
- metaflow-stubs/plugins/parallel_decorator.pyi +3 -3
- metaflow-stubs/plugins/perimeters.pyi +2 -2
- metaflow-stubs/plugins/project_decorator.pyi +3 -3
- metaflow-stubs/plugins/pypi/__init__.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +3 -3
- metaflow-stubs/plugins/pypi/utils.pyi +2 -2
- metaflow-stubs/plugins/resources_decorator.pyi +3 -3
- metaflow-stubs/plugins/retry_decorator.pyi +2 -2
- metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
- metaflow-stubs/plugins/storage_executor.pyi +2 -2
- metaflow-stubs/plugins/tag_cli.pyi +4 -4
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
- metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
- metaflow-stubs/procpoll.pyi +2 -2
- metaflow-stubs/profilers/__init__.pyi +2 -2
- metaflow-stubs/pylint_wrapper.pyi +2 -2
- metaflow-stubs/runner/__init__.pyi +2 -2
- metaflow-stubs/runner/deployer.pyi +4 -4
- metaflow-stubs/runner/metaflow_runner.pyi +5 -33
- metaflow-stubs/runner/nbdeploy.pyi +2 -2
- metaflow-stubs/runner/nbrun.pyi +2 -2
- metaflow-stubs/runner/subprocess_manager.pyi +2 -2
- metaflow-stubs/runner/utils.pyi +2 -2
- metaflow-stubs/system/__init__.pyi +4 -4
- metaflow-stubs/system/system_logger.pyi +3 -3
- metaflow-stubs/system/system_monitor.pyi +3 -3
- metaflow-stubs/tagging_util.pyi +2 -2
- metaflow-stubs/tuple_util.pyi +2 -2
- {ob_metaflow_stubs-6.0.3.114.dist-info → ob_metaflow_stubs-6.0.3.115.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.115.dist-info/RECORD +140 -0
- ob_metaflow_stubs-6.0.3.114.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.114.dist-info → ob_metaflow_stubs-6.0.3.115.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.114.dist-info → ob_metaflow_stubs-6.0.3.115.dist-info}/top_level.txt +0 -0
metaflow-stubs/__init__.pyi
CHANGED
@@ -1,24 +1,24 @@
|
|
1
1
|
##################################################################################
|
2
2
|
# Auto-generated Metaflow stub file #
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-10-
|
3
|
+
# MF version: 2.12.27.1+obcheckpoint(0.1.1);ob(v1) #
|
4
|
+
# Generated on 2024-10-25T21:39:54.730102 #
|
5
5
|
##################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import
|
12
|
-
import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
|
13
|
-
import metaflow.datastore.inputs
|
14
|
-
import metaflow.parameters
|
15
|
-
import metaflow.events
|
16
|
-
import metaflow.runner.metaflow_runner
|
17
|
-
import metaflow.client.core
|
11
|
+
import typing
|
18
12
|
import metaflow.metaflow_current
|
19
13
|
import datetime
|
20
14
|
import metaflow._vendor.click.types
|
21
|
-
import
|
15
|
+
import metaflow.runner.metaflow_runner
|
16
|
+
import metaflow.parameters
|
17
|
+
import metaflow.events
|
18
|
+
import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
|
19
|
+
import metaflow.client.core
|
20
|
+
import metaflow.datastore.inputs
|
21
|
+
import metaflow.flowspec
|
22
22
|
FlowSpecDerived = typing.TypeVar("FlowSpecDerived", bound="FlowSpec", contravariant=False, covariant=False)
|
23
23
|
StepFlag = typing.NewType("StepFlag", bool)
|
24
24
|
|
@@ -512,6 +512,56 @@ def step(f: typing.Union[typing.Callable[[FlowSpecDerived], None], typing.Callab
|
|
512
512
|
"""
|
513
513
|
...
|
514
514
|
|
515
|
+
@typing.overload
|
516
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
517
|
+
"""
|
518
|
+
Internal decorator to support Fast bakery
|
519
|
+
"""
|
520
|
+
...
|
521
|
+
|
522
|
+
@typing.overload
|
523
|
+
def fast_bakery_internal(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
524
|
+
...
|
525
|
+
|
526
|
+
def fast_bakery_internal(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None):
|
527
|
+
"""
|
528
|
+
Internal decorator to support Fast bakery
|
529
|
+
"""
|
530
|
+
...
|
531
|
+
|
532
|
+
@typing.overload
|
533
|
+
def secrets(*, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
534
|
+
"""
|
535
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
536
|
+
the execution of a step.
|
537
|
+
|
538
|
+
Parameters
|
539
|
+
----------
|
540
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
541
|
+
List of secret specs, defining how the secrets are to be retrieved
|
542
|
+
"""
|
543
|
+
...
|
544
|
+
|
545
|
+
@typing.overload
|
546
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
547
|
+
...
|
548
|
+
|
549
|
+
@typing.overload
|
550
|
+
def secrets(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
551
|
+
...
|
552
|
+
|
553
|
+
def secrets(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, sources: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = []):
|
554
|
+
"""
|
555
|
+
Specifies secrets to be retrieved and injected as environment variables prior to
|
556
|
+
the execution of a step.
|
557
|
+
|
558
|
+
Parameters
|
559
|
+
----------
|
560
|
+
sources : List[Union[str, Dict[str, Any]]], default: []
|
561
|
+
List of secret specs, defining how the secrets are to be retrieved
|
562
|
+
"""
|
563
|
+
...
|
564
|
+
|
515
565
|
@typing.overload
|
516
566
|
def checkpoint(*, load_policy: str = "fresh", temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
517
567
|
"""
|
@@ -574,228 +624,124 @@ def checkpoint(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None
|
|
574
624
|
...
|
575
625
|
|
576
626
|
@typing.overload
|
577
|
-
def
|
627
|
+
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
578
628
|
"""
|
579
|
-
|
580
|
-
|
629
|
+
Enables loading / saving of models within a step.
|
630
|
+
|
581
631
|
|
582
632
|
Parameters
|
583
633
|
----------
|
584
|
-
|
585
|
-
|
634
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
635
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
636
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
637
|
+
- `current.checkpoint`
|
638
|
+
- `current.model`
|
639
|
+
- `current.huggingface_hub`
|
640
|
+
|
641
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
642
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
643
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
644
|
+
|
645
|
+
temp_dir_root : str, default: None
|
646
|
+
The root directory under which `current.model.loaded` will store loaded models
|
647
|
+
|
648
|
+
|
649
|
+
|
586
650
|
"""
|
587
651
|
...
|
588
652
|
|
589
653
|
@typing.overload
|
590
|
-
def
|
654
|
+
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
591
655
|
...
|
592
656
|
|
593
657
|
@typing.overload
|
594
|
-
def
|
658
|
+
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
595
659
|
...
|
596
660
|
|
597
|
-
def
|
661
|
+
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
598
662
|
"""
|
599
|
-
|
600
|
-
the execution of a step.
|
663
|
+
Enables loading / saving of models within a step.
|
601
664
|
|
602
|
-
Parameters
|
603
|
-
----------
|
604
|
-
sources : List[Union[str, Dict[str, Any]]], default: []
|
605
|
-
List of secret specs, defining how the secrets are to be retrieved
|
606
|
-
"""
|
607
|
-
...
|
608
|
-
|
609
|
-
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
610
|
-
"""
|
611
|
-
Specifies that this step should execute on Kubernetes.
|
612
665
|
|
613
666
|
Parameters
|
614
667
|
----------
|
615
|
-
|
616
|
-
|
617
|
-
|
618
|
-
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
623
|
-
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
632
|
-
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
633
|
-
Kubernetes service account to use when launching pod in Kubernetes.
|
634
|
-
secrets : List[str], optional, default None
|
635
|
-
Kubernetes secrets to use when launching pod in Kubernetes. These
|
636
|
-
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
637
|
-
in Metaflow configuration.
|
638
|
-
node_selector: Union[Dict[str,str], str], optional, default None
|
639
|
-
Kubernetes node selector(s) to apply to the pod running the task.
|
640
|
-
Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
|
641
|
-
or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
|
642
|
-
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
643
|
-
Kubernetes namespace to use when launching pod in Kubernetes.
|
644
|
-
gpu : int, optional, default None
|
645
|
-
Number of GPUs required for this step. A value of zero implies that
|
646
|
-
the scheduled node should not have GPUs.
|
647
|
-
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
648
|
-
The vendor of the GPUs to be used for this step.
|
649
|
-
tolerations : List[str], default []
|
650
|
-
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
651
|
-
Kubernetes tolerations to use when launching pod in Kubernetes.
|
652
|
-
use_tmpfs : bool, default False
|
653
|
-
This enables an explicit tmpfs mount for this step.
|
654
|
-
tmpfs_tempdir : bool, default True
|
655
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
656
|
-
tmpfs_size : int, optional, default: None
|
657
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
658
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
659
|
-
memory allocated for this step.
|
660
|
-
tmpfs_path : str, optional, default /metaflow_temp
|
661
|
-
Path to tmpfs mount for this step.
|
662
|
-
persistent_volume_claims : Dict[str, str], optional, default None
|
663
|
-
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
664
|
-
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
665
|
-
shared_memory: int, optional
|
666
|
-
Shared memory size (in MiB) required for this step
|
667
|
-
port: int, optional
|
668
|
-
Port number to specify in the Kubernetes job object
|
669
|
-
compute_pool : str, optional, default None
|
670
|
-
Compute pool to be used for for this step.
|
671
|
-
If not specified, any accessible compute pool within the perimeter is used.
|
672
|
-
hostname_resolution_timeout: int, default 10 * 60
|
673
|
-
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
674
|
-
Only applicable when @parallel is used.
|
668
|
+
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
669
|
+
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
670
|
+
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
671
|
+
- `current.checkpoint`
|
672
|
+
- `current.model`
|
673
|
+
- `current.huggingface_hub`
|
674
|
+
|
675
|
+
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
676
|
+
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
677
|
+
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
678
|
+
|
679
|
+
temp_dir_root : str, default: None
|
680
|
+
The root directory under which `current.model.loaded` will store loaded models
|
681
|
+
|
682
|
+
|
683
|
+
|
675
684
|
"""
|
676
685
|
...
|
677
686
|
|
678
687
|
@typing.overload
|
679
|
-
def
|
688
|
+
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
680
689
|
"""
|
681
|
-
|
690
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
682
691
|
|
683
|
-
|
684
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
685
|
-
you can use `@conda_base` to set packages required by all
|
686
|
-
steps and use `@conda` to specify step-specific overrides.
|
692
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
687
693
|
|
688
694
|
Parameters
|
689
695
|
----------
|
690
|
-
|
691
|
-
|
692
|
-
|
693
|
-
|
694
|
-
|
695
|
-
|
696
|
-
|
697
|
-
|
698
|
-
|
699
|
-
|
696
|
+
type : str, default 'default'
|
697
|
+
Card type.
|
698
|
+
id : str, optional, default None
|
699
|
+
If multiple cards are present, use this id to identify this card.
|
700
|
+
options : Dict[str, Any], default {}
|
701
|
+
Options passed to the card. The contents depend on the card type.
|
702
|
+
timeout : int, default 45
|
703
|
+
Interrupt reporting if it takes more than this many seconds.
|
704
|
+
|
705
|
+
|
700
706
|
"""
|
701
707
|
...
|
702
708
|
|
703
709
|
@typing.overload
|
704
|
-
def
|
710
|
+
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
705
711
|
...
|
706
712
|
|
707
713
|
@typing.overload
|
708
|
-
def
|
714
|
+
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
709
715
|
...
|
710
716
|
|
711
|
-
def
|
717
|
+
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
712
718
|
"""
|
713
|
-
|
719
|
+
Creates a human-readable report, a Metaflow Card, after this step completes.
|
714
720
|
|
715
|
-
|
716
|
-
attributes set in the `@conda_base` flow-level decorator. Hence,
|
717
|
-
you can use `@conda_base` to set packages required by all
|
718
|
-
steps and use `@conda` to specify step-specific overrides.
|
721
|
+
Note that you may add multiple `@card` decorators in a step with different parameters.
|
719
722
|
|
720
723
|
Parameters
|
721
724
|
----------
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
|
725
|
+
type : str, default 'default'
|
726
|
+
Card type.
|
727
|
+
id : str, optional, default None
|
728
|
+
If multiple cards are present, use this id to identify this card.
|
729
|
+
options : Dict[str, Any], default {}
|
730
|
+
Options passed to the card. The contents depend on the card type.
|
731
|
+
timeout : int, default 45
|
732
|
+
Interrupt reporting if it takes more than this many seconds.
|
733
|
+
|
734
|
+
|
732
735
|
"""
|
733
736
|
...
|
734
737
|
|
735
738
|
@typing.overload
|
736
|
-
def
|
739
|
+
def resources(*, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
737
740
|
"""
|
738
|
-
Specifies
|
741
|
+
Specifies the resources needed when executing this step.
|
739
742
|
|
740
|
-
|
741
|
-
|
742
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
743
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
744
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
745
|
-
|
746
|
-
Note that all the values specified in parameters are added together so if you specify
|
747
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
748
|
-
|
749
|
-
Parameters
|
750
|
-
----------
|
751
|
-
seconds : int, default 0
|
752
|
-
Number of seconds to wait prior to timing out.
|
753
|
-
minutes : int, default 0
|
754
|
-
Number of minutes to wait prior to timing out.
|
755
|
-
hours : int, default 0
|
756
|
-
Number of hours to wait prior to timing out.
|
757
|
-
"""
|
758
|
-
...
|
759
|
-
|
760
|
-
@typing.overload
|
761
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
762
|
-
...
|
763
|
-
|
764
|
-
@typing.overload
|
765
|
-
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
766
|
-
...
|
767
|
-
|
768
|
-
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
769
|
-
"""
|
770
|
-
Specifies a timeout for your step.
|
771
|
-
|
772
|
-
This decorator is useful if this step may hang indefinitely.
|
773
|
-
|
774
|
-
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
775
|
-
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
776
|
-
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
777
|
-
|
778
|
-
Note that all the values specified in parameters are added together so if you specify
|
779
|
-
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
780
|
-
|
781
|
-
Parameters
|
782
|
-
----------
|
783
|
-
seconds : int, default 0
|
784
|
-
Number of seconds to wait prior to timing out.
|
785
|
-
minutes : int, default 0
|
786
|
-
Number of minutes to wait prior to timing out.
|
787
|
-
hours : int, default 0
|
788
|
-
Number of hours to wait prior to timing out.
|
789
|
-
"""
|
790
|
-
...
|
791
|
-
|
792
|
-
@typing.overload
|
793
|
-
def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
794
|
-
"""
|
795
|
-
Specifies the resources needed when executing this step.
|
796
|
-
|
797
|
-
Use `@resources` to specify the resource requirements
|
798
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
743
|
+
Use `@resources` to specify the resource requirements
|
744
|
+
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
799
745
|
|
800
746
|
You can choose the compute layer on the command line by executing e.g.
|
801
747
|
```
|
@@ -812,7 +758,7 @@ def resources(*, cpu: int = 1, gpu: int = 0, disk: typing.Optional[int] = None,
|
|
812
758
|
----------
|
813
759
|
cpu : int, default 1
|
814
760
|
Number of CPUs required for this step.
|
815
|
-
gpu : int, default
|
761
|
+
gpu : int, optional, default None
|
816
762
|
Number of GPUs required for this step.
|
817
763
|
disk : int, optional, default None
|
818
764
|
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
@@ -832,7 +778,7 @@ def resources(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.C
|
|
832
778
|
def resources(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
833
779
|
...
|
834
780
|
|
835
|
-
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: int =
|
781
|
+
def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, cpu: int = 1, gpu: typing.Optional[int] = None, disk: typing.Optional[int] = None, memory: int = 4096, shared_memory: typing.Optional[int] = None):
|
836
782
|
"""
|
837
783
|
Specifies the resources needed when executing this step.
|
838
784
|
|
@@ -854,7 +800,7 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
854
800
|
----------
|
855
801
|
cpu : int, default 1
|
856
802
|
Number of CPUs required for this step.
|
857
|
-
gpu : int, default
|
803
|
+
gpu : int, optional, default None
|
858
804
|
Number of GPUs required for this step.
|
859
805
|
disk : int, optional, default None
|
860
806
|
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
@@ -867,19 +813,90 @@ def resources(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None]
|
|
867
813
|
...
|
868
814
|
|
869
815
|
@typing.overload
|
870
|
-
def
|
816
|
+
def conda(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
871
817
|
"""
|
872
|
-
|
818
|
+
Specifies the Conda environment for the step.
|
819
|
+
|
820
|
+
Information in this decorator will augment any
|
821
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
822
|
+
you can use `@conda_base` to set packages required by all
|
823
|
+
steps and use `@conda` to specify step-specific overrides.
|
824
|
+
|
825
|
+
Parameters
|
826
|
+
----------
|
827
|
+
packages : Dict[str, str], default {}
|
828
|
+
Packages to use for this step. The key is the name of the package
|
829
|
+
and the value is the version to use.
|
830
|
+
libraries : Dict[str, str], default {}
|
831
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
832
|
+
python : str, optional, default None
|
833
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
834
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
835
|
+
disabled : bool, default False
|
836
|
+
If set to True, disables @conda.
|
873
837
|
"""
|
874
838
|
...
|
875
839
|
|
876
840
|
@typing.overload
|
877
|
-
def
|
841
|
+
def conda(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
878
842
|
...
|
879
843
|
|
880
|
-
|
844
|
+
@typing.overload
|
845
|
+
def conda(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
846
|
+
...
|
847
|
+
|
848
|
+
def conda(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
881
849
|
"""
|
882
|
-
|
850
|
+
Specifies the Conda environment for the step.
|
851
|
+
|
852
|
+
Information in this decorator will augment any
|
853
|
+
attributes set in the `@conda_base` flow-level decorator. Hence,
|
854
|
+
you can use `@conda_base` to set packages required by all
|
855
|
+
steps and use `@conda` to specify step-specific overrides.
|
856
|
+
|
857
|
+
Parameters
|
858
|
+
----------
|
859
|
+
packages : Dict[str, str], default {}
|
860
|
+
Packages to use for this step. The key is the name of the package
|
861
|
+
and the value is the version to use.
|
862
|
+
libraries : Dict[str, str], default {}
|
863
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
864
|
+
python : str, optional, default None
|
865
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
866
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
867
|
+
disabled : bool, default False
|
868
|
+
If set to True, disables @conda.
|
869
|
+
"""
|
870
|
+
...
|
871
|
+
|
872
|
+
@typing.overload
|
873
|
+
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
874
|
+
"""
|
875
|
+
Specifies environment variables to be set prior to the execution of a step.
|
876
|
+
|
877
|
+
Parameters
|
878
|
+
----------
|
879
|
+
vars : Dict[str, str], default {}
|
880
|
+
Dictionary of environment variables to set.
|
881
|
+
"""
|
882
|
+
...
|
883
|
+
|
884
|
+
@typing.overload
|
885
|
+
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
886
|
+
...
|
887
|
+
|
888
|
+
@typing.overload
|
889
|
+
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
890
|
+
...
|
891
|
+
|
892
|
+
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
893
|
+
"""
|
894
|
+
Specifies environment variables to be set prior to the execution of a step.
|
895
|
+
|
896
|
+
Parameters
|
897
|
+
----------
|
898
|
+
vars : Dict[str, str], default {}
|
899
|
+
Dictionary of environment variables to set.
|
883
900
|
"""
|
884
901
|
...
|
885
902
|
|
@@ -932,57 +949,6 @@ def catch(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
932
949
|
"""
|
933
950
|
...
|
934
951
|
|
935
|
-
@typing.overload
|
936
|
-
def card(*, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
937
|
-
"""
|
938
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
939
|
-
|
940
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
941
|
-
|
942
|
-
Parameters
|
943
|
-
----------
|
944
|
-
type : str, default 'default'
|
945
|
-
Card type.
|
946
|
-
id : str, optional, default None
|
947
|
-
If multiple cards are present, use this id to identify this card.
|
948
|
-
options : Dict[str, Any], default {}
|
949
|
-
Options passed to the card. The contents depend on the card type.
|
950
|
-
timeout : int, default 45
|
951
|
-
Interrupt reporting if it takes more than this many seconds.
|
952
|
-
|
953
|
-
|
954
|
-
"""
|
955
|
-
...
|
956
|
-
|
957
|
-
@typing.overload
|
958
|
-
def card(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
959
|
-
...
|
960
|
-
|
961
|
-
@typing.overload
|
962
|
-
def card(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
963
|
-
...
|
964
|
-
|
965
|
-
def card(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, type: str = "default", id: typing.Optional[str] = None, options: typing.Dict[str, typing.Any] = {}, timeout: int = 45):
|
966
|
-
"""
|
967
|
-
Creates a human-readable report, a Metaflow Card, after this step completes.
|
968
|
-
|
969
|
-
Note that you may add multiple `@card` decorators in a step with different parameters.
|
970
|
-
|
971
|
-
Parameters
|
972
|
-
----------
|
973
|
-
type : str, default 'default'
|
974
|
-
Card type.
|
975
|
-
id : str, optional, default None
|
976
|
-
If multiple cards are present, use this id to identify this card.
|
977
|
-
options : Dict[str, Any], default {}
|
978
|
-
Options passed to the card. The contents depend on the card type.
|
979
|
-
timeout : int, default 45
|
980
|
-
Interrupt reporting if it takes more than this many seconds.
|
981
|
-
|
982
|
-
|
983
|
-
"""
|
984
|
-
...
|
985
|
-
|
986
952
|
@typing.overload
|
987
953
|
def parallel(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
988
954
|
"""
|
@@ -1002,37 +968,6 @@ def parallel(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None],
|
|
1002
968
|
"""
|
1003
969
|
...
|
1004
970
|
|
1005
|
-
@typing.overload
|
1006
|
-
def environment(*, vars: typing.Dict[str, str] = {}) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1007
|
-
"""
|
1008
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1009
|
-
|
1010
|
-
Parameters
|
1011
|
-
----------
|
1012
|
-
vars : Dict[str, str], default {}
|
1013
|
-
Dictionary of environment variables to set.
|
1014
|
-
"""
|
1015
|
-
...
|
1016
|
-
|
1017
|
-
@typing.overload
|
1018
|
-
def environment(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1019
|
-
...
|
1020
|
-
|
1021
|
-
@typing.overload
|
1022
|
-
def environment(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1023
|
-
...
|
1024
|
-
|
1025
|
-
def environment(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, vars: typing.Dict[str, str] = {}):
|
1026
|
-
"""
|
1027
|
-
Specifies environment variables to be set prior to the execution of a step.
|
1028
|
-
|
1029
|
-
Parameters
|
1030
|
-
----------
|
1031
|
-
vars : Dict[str, str], default {}
|
1032
|
-
Dictionary of environment variables to set.
|
1033
|
-
"""
|
1034
|
-
...
|
1035
|
-
|
1036
971
|
@typing.overload
|
1037
972
|
def pypi(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1038
973
|
"""
|
@@ -1082,6 +1017,132 @@ def pypi(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typ
|
|
1082
1017
|
"""
|
1083
1018
|
...
|
1084
1019
|
|
1020
|
+
@typing.overload
|
1021
|
+
def timeout(*, seconds: int = 0, minutes: int = 0, hours: int = 0) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1022
|
+
"""
|
1023
|
+
Specifies a timeout for your step.
|
1024
|
+
|
1025
|
+
This decorator is useful if this step may hang indefinitely.
|
1026
|
+
|
1027
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1028
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1029
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1030
|
+
|
1031
|
+
Note that all the values specified in parameters are added together so if you specify
|
1032
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1033
|
+
|
1034
|
+
Parameters
|
1035
|
+
----------
|
1036
|
+
seconds : int, default 0
|
1037
|
+
Number of seconds to wait prior to timing out.
|
1038
|
+
minutes : int, default 0
|
1039
|
+
Number of minutes to wait prior to timing out.
|
1040
|
+
hours : int, default 0
|
1041
|
+
Number of hours to wait prior to timing out.
|
1042
|
+
"""
|
1043
|
+
...
|
1044
|
+
|
1045
|
+
@typing.overload
|
1046
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1047
|
+
...
|
1048
|
+
|
1049
|
+
@typing.overload
|
1050
|
+
def timeout(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1051
|
+
...
|
1052
|
+
|
1053
|
+
def timeout(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, seconds: int = 0, minutes: int = 0, hours: int = 0):
|
1054
|
+
"""
|
1055
|
+
Specifies a timeout for your step.
|
1056
|
+
|
1057
|
+
This decorator is useful if this step may hang indefinitely.
|
1058
|
+
|
1059
|
+
This can be used in conjunction with the `@retry` decorator as well as the `@catch` decorator.
|
1060
|
+
A timeout is considered to be an exception thrown by the step. It will cause the step to be
|
1061
|
+
retried if needed and the exception will be caught by the `@catch` decorator, if present.
|
1062
|
+
|
1063
|
+
Note that all the values specified in parameters are added together so if you specify
|
1064
|
+
60 seconds and 1 hour, the decorator will have an effective timeout of 1 hour and 1 minute.
|
1065
|
+
|
1066
|
+
Parameters
|
1067
|
+
----------
|
1068
|
+
seconds : int, default 0
|
1069
|
+
Number of seconds to wait prior to timing out.
|
1070
|
+
minutes : int, default 0
|
1071
|
+
Number of minutes to wait prior to timing out.
|
1072
|
+
hours : int, default 0
|
1073
|
+
Number of hours to wait prior to timing out.
|
1074
|
+
"""
|
1075
|
+
...
|
1076
|
+
|
1077
|
+
def kubernetes(*, cpu: int = 1, memory: int = 4096, disk: int = 10240, image: typing.Optional[str] = None, image_pull_policy: str = "KUBERNETES_IMAGE_PULL_POLICY", service_account: str = "METAFLOW_KUBERNETES_SERVICE_ACCOUNT", secrets: typing.Optional[typing.List[str]] = None, node_selector: typing.Union[typing.Dict[str, str], str, None] = None, namespace: str = "METAFLOW_KUBERNETES_NAMESPACE", gpu: typing.Optional[int] = None, gpu_vendor: str = "KUBERNETES_GPU_VENDOR", tolerations: typing.List[str] = [], use_tmpfs: bool = False, tmpfs_tempdir: bool = True, tmpfs_size: typing.Optional[int] = None, tmpfs_path: typing.Optional[str] = "/metaflow_temp", persistent_volume_claims: typing.Optional[typing.Dict[str, str]] = None, shared_memory: typing.Optional[int] = None, port: typing.Optional[int] = None, compute_pool: typing.Optional[str] = None, hostname_resolution_timeout: int = 600) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1078
|
+
"""
|
1079
|
+
Specifies that this step should execute on Kubernetes.
|
1080
|
+
|
1081
|
+
Parameters
|
1082
|
+
----------
|
1083
|
+
cpu : int, default 1
|
1084
|
+
Number of CPUs required for this step. If `@resources` is
|
1085
|
+
also present, the maximum value from all decorators is used.
|
1086
|
+
memory : int, default 4096
|
1087
|
+
Memory size (in MB) required for this step. If
|
1088
|
+
`@resources` is also present, the maximum value from all decorators is
|
1089
|
+
used.
|
1090
|
+
disk : int, default 10240
|
1091
|
+
Disk size (in MB) required for this step. If
|
1092
|
+
`@resources` is also present, the maximum value from all decorators is
|
1093
|
+
used.
|
1094
|
+
image : str, optional, default None
|
1095
|
+
Docker image to use when launching on Kubernetes. If not specified, and
|
1096
|
+
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
1097
|
+
not, a default Docker image mapping to the current version of Python is used.
|
1098
|
+
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
1099
|
+
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
1100
|
+
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
1101
|
+
Kubernetes service account to use when launching pod in Kubernetes.
|
1102
|
+
secrets : List[str], optional, default None
|
1103
|
+
Kubernetes secrets to use when launching pod in Kubernetes. These
|
1104
|
+
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
1105
|
+
in Metaflow configuration.
|
1106
|
+
node_selector: Union[Dict[str,str], str], optional, default None
|
1107
|
+
Kubernetes node selector(s) to apply to the pod running the task.
|
1108
|
+
Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
|
1109
|
+
or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
|
1110
|
+
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
1111
|
+
Kubernetes namespace to use when launching pod in Kubernetes.
|
1112
|
+
gpu : int, optional, default None
|
1113
|
+
Number of GPUs required for this step. A value of zero implies that
|
1114
|
+
the scheduled node should not have GPUs.
|
1115
|
+
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
1116
|
+
The vendor of the GPUs to be used for this step.
|
1117
|
+
tolerations : List[str], default []
|
1118
|
+
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
1119
|
+
Kubernetes tolerations to use when launching pod in Kubernetes.
|
1120
|
+
use_tmpfs : bool, default False
|
1121
|
+
This enables an explicit tmpfs mount for this step.
|
1122
|
+
tmpfs_tempdir : bool, default True
|
1123
|
+
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
1124
|
+
tmpfs_size : int, optional, default: None
|
1125
|
+
The value for the size (in MiB) of the tmpfs mount for this step.
|
1126
|
+
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
1127
|
+
memory allocated for this step.
|
1128
|
+
tmpfs_path : str, optional, default /metaflow_temp
|
1129
|
+
Path to tmpfs mount for this step.
|
1130
|
+
persistent_volume_claims : Dict[str, str], optional, default None
|
1131
|
+
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
1132
|
+
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
1133
|
+
shared_memory: int, optional
|
1134
|
+
Shared memory size (in MiB) required for this step
|
1135
|
+
port: int, optional
|
1136
|
+
Port number to specify in the Kubernetes job object
|
1137
|
+
compute_pool : str, optional, default None
|
1138
|
+
Compute pool to be used for for this step.
|
1139
|
+
If not specified, any accessible compute pool within the perimeter is used.
|
1140
|
+
hostname_resolution_timeout: int, default 10 * 60
|
1141
|
+
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
1142
|
+
Only applicable when @parallel is used.
|
1143
|
+
"""
|
1144
|
+
...
|
1145
|
+
|
1085
1146
|
def huggingface_hub(*, temp_dir_root: typing.Optional[str] = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1086
1147
|
"""
|
1087
1148
|
Decorator that helps cache, version and store models/datasets from huggingface hub.
|
@@ -1148,64 +1209,34 @@ def retry(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], ty
|
|
1148
1209
|
"""
|
1149
1210
|
...
|
1150
1211
|
|
1151
|
-
|
1152
|
-
def model(*, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None) -> typing.Callable[[typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]], typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]]]:
|
1212
|
+
def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1153
1213
|
"""
|
1154
|
-
|
1155
|
-
|
1156
|
-
|
1157
|
-
Parameters
|
1158
|
-
----------
|
1159
|
-
load : Union[List[str],str,List[Tuple[str,Union[str,None]]]], default: None
|
1160
|
-
Artifact name/s referencing the models/checkpoints to load. Artifact names refer to the names of the instance variables set to `self`.
|
1161
|
-
These artifact names give to `load` be reference objects or reference `key` string's from objects created by:
|
1162
|
-
- `current.checkpoint`
|
1163
|
-
- `current.model`
|
1164
|
-
- `current.huggingface_hub`
|
1165
|
-
|
1166
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
1167
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
1168
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
1169
|
-
|
1170
|
-
temp_dir_root : str, default: None
|
1171
|
-
The root directory under which `current.model.loaded` will store loaded models
|
1172
|
-
|
1214
|
+
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
1173
1215
|
|
1216
|
+
User code call
|
1217
|
+
-----------
|
1218
|
+
@nim(
|
1219
|
+
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
1220
|
+
backend='managed'
|
1221
|
+
)
|
1174
1222
|
|
1175
|
-
|
1176
|
-
|
1177
|
-
|
1178
|
-
|
1179
|
-
def model(f: typing.Callable[[FlowSpecDerived, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, StepFlag], None]:
|
1180
|
-
...
|
1181
|
-
|
1182
|
-
@typing.overload
|
1183
|
-
def model(f: typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]) -> typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None]:
|
1184
|
-
...
|
1185
|
-
|
1186
|
-
def model(f: typing.Union[typing.Callable[[FlowSpecDerived, StepFlag], None], typing.Callable[[FlowSpecDerived, typing.Any, StepFlag], None], None] = None, *, load: typing.Union[typing.List[str], str, typing.List[typing.Tuple[str, typing.Optional[str]]]] = None, temp_dir_root: str = None):
|
1187
|
-
"""
|
1188
|
-
Enables loading / saving of models within a step.
|
1223
|
+
Valid backend options
|
1224
|
+
---------------------
|
1225
|
+
- 'managed': Outerbounds selects a compute provider based on the model.
|
1226
|
+
- 🚧 'dataplane': Run in your account.
|
1189
1227
|
|
1228
|
+
Valid model options
|
1229
|
+
----------------
|
1230
|
+
- 'meta/llama3-8b-instruct': 8B parameter model
|
1231
|
+
- 'meta/llama3-70b-instruct': 70B parameter model
|
1232
|
+
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
1190
1233
|
|
1191
1234
|
Parameters
|
1192
1235
|
----------
|
1193
|
-
|
1194
|
-
|
1195
|
-
|
1196
|
-
|
1197
|
-
- `current.model`
|
1198
|
-
- `current.huggingface_hub`
|
1199
|
-
|
1200
|
-
If a list of tuples is provided, the first element is the artifact name and the second element is the path the artifact needs be unpacked on
|
1201
|
-
the local filesystem. If the second element is None, the artifact will be unpacked in the current working directory.
|
1202
|
-
If a string is provided, then the artifact corresponding to that name will be loaded in the current working directory.
|
1203
|
-
|
1204
|
-
temp_dir_root : str, default: None
|
1205
|
-
The root directory under which `current.model.loaded` will store loaded models
|
1206
|
-
|
1207
|
-
|
1208
|
-
|
1236
|
+
models: list[NIM]
|
1237
|
+
List of NIM containers running models in sidecars.
|
1238
|
+
backend: str
|
1239
|
+
Compute provider to run the NIM container.
|
1209
1240
|
"""
|
1210
1241
|
...
|
1211
1242
|
|
@@ -1278,84 +1309,35 @@ def trigger_on_finish(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *
|
|
1278
1309
|
when upstream runs within the same namespace complete successfully
|
1279
1310
|
|
1280
1311
|
Additionally, you can specify project aware upstream flow dependencies
|
1281
|
-
by specifying the fully qualified project_flow_name.
|
1282
|
-
```
|
1283
|
-
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1284
|
-
```
|
1285
|
-
or
|
1286
|
-
```
|
1287
|
-
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1288
|
-
```
|
1289
|
-
|
1290
|
-
You can also specify just the project or project branch (other values will be
|
1291
|
-
inferred from the current project or project branch):
|
1292
|
-
```
|
1293
|
-
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1294
|
-
```
|
1295
|
-
|
1296
|
-
Note that `branch` is typically one of:
|
1297
|
-
- `prod`
|
1298
|
-
- `user.bob`
|
1299
|
-
- `test.my_experiment`
|
1300
|
-
- `prod.staging`
|
1301
|
-
|
1302
|
-
Parameters
|
1303
|
-
----------
|
1304
|
-
flow : Union[str, Dict[str, str]], optional, default None
|
1305
|
-
Upstream flow dependency for this flow.
|
1306
|
-
flows : List[Union[str, Dict[str, str]]], default []
|
1307
|
-
Upstream flow dependencies for this flow.
|
1308
|
-
options : Dict[str, Any], default {}
|
1309
|
-
Backend-specific configuration for tuning eventing behavior.
|
1310
|
-
|
1311
|
-
|
1312
|
-
"""
|
1313
|
-
...
|
1314
|
-
|
1315
|
-
def nim(*, models: "list[NIM]", backend: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1316
|
-
"""
|
1317
|
-
This decorator is used to run NIM containers in Metaflow tasks as sidecars.
|
1318
|
-
|
1319
|
-
User code call
|
1320
|
-
-----------
|
1321
|
-
@nim(
|
1322
|
-
models=['meta/llama3-8b-instruct', 'meta/llama3-70b-instruct'],
|
1323
|
-
backend='managed'
|
1324
|
-
)
|
1325
|
-
|
1326
|
-
Valid backend options
|
1327
|
-
---------------------
|
1328
|
-
- 'managed': Outerbounds selects a compute provider based on the model.
|
1329
|
-
- 🚧 'dataplane': Run in your account.
|
1330
|
-
|
1331
|
-
Valid model options
|
1332
|
-
----------------
|
1333
|
-
- 'meta/llama3-8b-instruct': 8B parameter model
|
1334
|
-
- 'meta/llama3-70b-instruct': 70B parameter model
|
1335
|
-
- Upon request, any model here: https://nvcf.ngc.nvidia.com/functions?filter=nvidia-functions
|
1312
|
+
by specifying the fully qualified project_flow_name.
|
1313
|
+
```
|
1314
|
+
@trigger_on_finish(flow='my_project.branch.my_branch.FooFlow')
|
1315
|
+
```
|
1316
|
+
or
|
1317
|
+
```
|
1318
|
+
@trigger_on_finish(flows=['my_project.branch.my_branch.FooFlow', 'BarFlow'])
|
1319
|
+
```
|
1336
1320
|
|
1337
|
-
|
1338
|
-
|
1339
|
-
|
1340
|
-
|
1341
|
-
|
1342
|
-
Compute provider to run the NIM container.
|
1343
|
-
"""
|
1344
|
-
...
|
1345
|
-
|
1346
|
-
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1347
|
-
"""
|
1348
|
-
Specifies what flows belong to the same project.
|
1321
|
+
You can also specify just the project or project branch (other values will be
|
1322
|
+
inferred from the current project or project branch):
|
1323
|
+
```
|
1324
|
+
@trigger_on_finish(flow={"name": "FooFlow", "project": "my_project", "project_branch": "branch"})
|
1325
|
+
```
|
1349
1326
|
|
1350
|
-
|
1351
|
-
|
1327
|
+
Note that `branch` is typically one of:
|
1328
|
+
- `prod`
|
1329
|
+
- `user.bob`
|
1330
|
+
- `test.my_experiment`
|
1331
|
+
- `prod.staging`
|
1352
1332
|
|
1353
1333
|
Parameters
|
1354
1334
|
----------
|
1355
|
-
|
1356
|
-
|
1357
|
-
|
1358
|
-
|
1335
|
+
flow : Union[str, Dict[str, str]], optional, default None
|
1336
|
+
Upstream flow dependency for this flow.
|
1337
|
+
flows : List[Union[str, Dict[str, str]]], default []
|
1338
|
+
Upstream flow dependencies for this flow.
|
1339
|
+
options : Dict[str, Any], default {}
|
1340
|
+
Backend-specific configuration for tuning eventing behavior.
|
1359
1341
|
|
1360
1342
|
|
1361
1343
|
"""
|
@@ -1403,104 +1385,6 @@ def airflow_external_task_sensor(*, timeout: int, poke_interval: int, mode: str,
|
|
1403
1385
|
"""
|
1404
1386
|
...
|
1405
1387
|
|
1406
|
-
@typing.overload
|
1407
|
-
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1408
|
-
"""
|
1409
|
-
Specifies the Conda environment for all steps of the flow.
|
1410
|
-
|
1411
|
-
Use `@conda_base` to set common libraries required by all
|
1412
|
-
steps and use `@conda` to specify step-specific additions.
|
1413
|
-
|
1414
|
-
Parameters
|
1415
|
-
----------
|
1416
|
-
packages : Dict[str, str], default {}
|
1417
|
-
Packages to use for this flow. The key is the name of the package
|
1418
|
-
and the value is the version to use.
|
1419
|
-
libraries : Dict[str, str], default {}
|
1420
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1421
|
-
python : str, optional, default None
|
1422
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1423
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1424
|
-
disabled : bool, default False
|
1425
|
-
If set to True, disables Conda.
|
1426
|
-
"""
|
1427
|
-
...
|
1428
|
-
|
1429
|
-
@typing.overload
|
1430
|
-
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1431
|
-
...
|
1432
|
-
|
1433
|
-
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1434
|
-
"""
|
1435
|
-
Specifies the Conda environment for all steps of the flow.
|
1436
|
-
|
1437
|
-
Use `@conda_base` to set common libraries required by all
|
1438
|
-
steps and use `@conda` to specify step-specific additions.
|
1439
|
-
|
1440
|
-
Parameters
|
1441
|
-
----------
|
1442
|
-
packages : Dict[str, str], default {}
|
1443
|
-
Packages to use for this flow. The key is the name of the package
|
1444
|
-
and the value is the version to use.
|
1445
|
-
libraries : Dict[str, str], default {}
|
1446
|
-
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1447
|
-
python : str, optional, default None
|
1448
|
-
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1449
|
-
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1450
|
-
disabled : bool, default False
|
1451
|
-
If set to True, disables Conda.
|
1452
|
-
"""
|
1453
|
-
...
|
1454
|
-
|
1455
|
-
@typing.overload
|
1456
|
-
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1457
|
-
"""
|
1458
|
-
Specifies the times when the flow should be run when running on a
|
1459
|
-
production scheduler.
|
1460
|
-
|
1461
|
-
Parameters
|
1462
|
-
----------
|
1463
|
-
hourly : bool, default False
|
1464
|
-
Run the workflow hourly.
|
1465
|
-
daily : bool, default True
|
1466
|
-
Run the workflow daily.
|
1467
|
-
weekly : bool, default False
|
1468
|
-
Run the workflow weekly.
|
1469
|
-
cron : str, optional, default None
|
1470
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1471
|
-
specified by this expression.
|
1472
|
-
timezone : str, optional, default None
|
1473
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1474
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1475
|
-
"""
|
1476
|
-
...
|
1477
|
-
|
1478
|
-
@typing.overload
|
1479
|
-
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1480
|
-
...
|
1481
|
-
|
1482
|
-
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1483
|
-
"""
|
1484
|
-
Specifies the times when the flow should be run when running on a
|
1485
|
-
production scheduler.
|
1486
|
-
|
1487
|
-
Parameters
|
1488
|
-
----------
|
1489
|
-
hourly : bool, default False
|
1490
|
-
Run the workflow hourly.
|
1491
|
-
daily : bool, default True
|
1492
|
-
Run the workflow daily.
|
1493
|
-
weekly : bool, default False
|
1494
|
-
Run the workflow weekly.
|
1495
|
-
cron : str, optional, default None
|
1496
|
-
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1497
|
-
specified by this expression.
|
1498
|
-
timezone : str, optional, default None
|
1499
|
-
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1500
|
-
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1501
|
-
"""
|
1502
|
-
...
|
1503
|
-
|
1504
1388
|
@typing.overload
|
1505
1389
|
def pypi_base(*, packages: typing.Dict[str, str] = {}, python: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1506
1390
|
"""
|
@@ -1582,6 +1466,55 @@ def airflow_s3_key_sensor(*, timeout: int, poke_interval: int, mode: str, expone
|
|
1582
1466
|
"""
|
1583
1467
|
...
|
1584
1468
|
|
1469
|
+
@typing.overload
|
1470
|
+
def conda_base(*, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1471
|
+
"""
|
1472
|
+
Specifies the Conda environment for all steps of the flow.
|
1473
|
+
|
1474
|
+
Use `@conda_base` to set common libraries required by all
|
1475
|
+
steps and use `@conda` to specify step-specific additions.
|
1476
|
+
|
1477
|
+
Parameters
|
1478
|
+
----------
|
1479
|
+
packages : Dict[str, str], default {}
|
1480
|
+
Packages to use for this flow. The key is the name of the package
|
1481
|
+
and the value is the version to use.
|
1482
|
+
libraries : Dict[str, str], default {}
|
1483
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1484
|
+
python : str, optional, default None
|
1485
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1486
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1487
|
+
disabled : bool, default False
|
1488
|
+
If set to True, disables Conda.
|
1489
|
+
"""
|
1490
|
+
...
|
1491
|
+
|
1492
|
+
@typing.overload
|
1493
|
+
def conda_base(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1494
|
+
...
|
1495
|
+
|
1496
|
+
def conda_base(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, packages: typing.Dict[str, str] = {}, libraries: typing.Dict[str, str] = {}, python: typing.Optional[str] = None, disabled: bool = False):
|
1497
|
+
"""
|
1498
|
+
Specifies the Conda environment for all steps of the flow.
|
1499
|
+
|
1500
|
+
Use `@conda_base` to set common libraries required by all
|
1501
|
+
steps and use `@conda` to specify step-specific additions.
|
1502
|
+
|
1503
|
+
Parameters
|
1504
|
+
----------
|
1505
|
+
packages : Dict[str, str], default {}
|
1506
|
+
Packages to use for this flow. The key is the name of the package
|
1507
|
+
and the value is the version to use.
|
1508
|
+
libraries : Dict[str, str], default {}
|
1509
|
+
Supported for backward compatibility. When used with packages, packages will take precedence.
|
1510
|
+
python : str, optional, default None
|
1511
|
+
Version of Python to use, e.g. '3.7.4'. A default value of None implies
|
1512
|
+
that the version used will correspond to the version of the Python interpreter used to start the run.
|
1513
|
+
disabled : bool, default False
|
1514
|
+
If set to True, disables Conda.
|
1515
|
+
"""
|
1516
|
+
...
|
1517
|
+
|
1585
1518
|
@typing.overload
|
1586
1519
|
def trigger(*, event: typing.Union[str, typing.Dict[str, typing.Any], None] = None, events: typing.List[typing.Union[str, typing.Dict[str, typing.Any]]] = [], options: typing.Dict[str, typing.Any] = {}) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1587
1520
|
"""
|
@@ -1677,6 +1610,73 @@ def trigger(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, event: t
|
|
1677
1610
|
"""
|
1678
1611
|
...
|
1679
1612
|
|
1613
|
+
def project(*, name: str) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1614
|
+
"""
|
1615
|
+
Specifies what flows belong to the same project.
|
1616
|
+
|
1617
|
+
A project-specific namespace is created for all flows that
|
1618
|
+
use the same `@project(name)`.
|
1619
|
+
|
1620
|
+
Parameters
|
1621
|
+
----------
|
1622
|
+
name : str
|
1623
|
+
Project name. Make sure that the name is unique amongst all
|
1624
|
+
projects that use the same production scheduler. The name may
|
1625
|
+
contain only lowercase alphanumeric characters and underscores.
|
1626
|
+
|
1627
|
+
|
1628
|
+
"""
|
1629
|
+
...
|
1630
|
+
|
1631
|
+
@typing.overload
|
1632
|
+
def schedule(*, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None) -> typing.Callable[[typing.Type[FlowSpecDerived]], typing.Type[FlowSpecDerived]]:
|
1633
|
+
"""
|
1634
|
+
Specifies the times when the flow should be run when running on a
|
1635
|
+
production scheduler.
|
1636
|
+
|
1637
|
+
Parameters
|
1638
|
+
----------
|
1639
|
+
hourly : bool, default False
|
1640
|
+
Run the workflow hourly.
|
1641
|
+
daily : bool, default True
|
1642
|
+
Run the workflow daily.
|
1643
|
+
weekly : bool, default False
|
1644
|
+
Run the workflow weekly.
|
1645
|
+
cron : str, optional, default None
|
1646
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1647
|
+
specified by this expression.
|
1648
|
+
timezone : str, optional, default None
|
1649
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1650
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1651
|
+
"""
|
1652
|
+
...
|
1653
|
+
|
1654
|
+
@typing.overload
|
1655
|
+
def schedule(f: typing.Type[FlowSpecDerived]) -> typing.Type[FlowSpecDerived]:
|
1656
|
+
...
|
1657
|
+
|
1658
|
+
def schedule(f: typing.Optional[typing.Type[FlowSpecDerived]] = None, *, hourly: bool = False, daily: bool = True, weekly: bool = False, cron: typing.Optional[str] = None, timezone: typing.Optional[str] = None):
|
1659
|
+
"""
|
1660
|
+
Specifies the times when the flow should be run when running on a
|
1661
|
+
production scheduler.
|
1662
|
+
|
1663
|
+
Parameters
|
1664
|
+
----------
|
1665
|
+
hourly : bool, default False
|
1666
|
+
Run the workflow hourly.
|
1667
|
+
daily : bool, default True
|
1668
|
+
Run the workflow daily.
|
1669
|
+
weekly : bool, default False
|
1670
|
+
Run the workflow weekly.
|
1671
|
+
cron : str, optional, default None
|
1672
|
+
Run the workflow at [a custom Cron schedule](https://docs.aws.amazon.com/eventbridge/latest/userguide/scheduled-events.html#cron-expressions)
|
1673
|
+
specified by this expression.
|
1674
|
+
timezone : str, optional, default None
|
1675
|
+
Timezone on which the schedule runs (default: None). Currently supported only for Argo workflows,
|
1676
|
+
which accepts timezones in [IANA format](https://nodatime.org/TimeZones).
|
1677
|
+
"""
|
1678
|
+
...
|
1679
|
+
|
1680
1680
|
def namespace(ns: typing.Optional[str]) -> typing.Optional[str]:
|
1681
1681
|
"""
|
1682
1682
|
Switch namespace to the one provided.
|
@@ -1768,7 +1768,7 @@ class Metaflow(object, metaclass=type):
|
|
1768
1768
|
flows present in the current namespace will be returned. A `Flow` is present in a namespace
|
1769
1769
|
if it has at least one run in the namespace.
|
1770
1770
|
"""
|
1771
|
-
def __init__(self):
|
1771
|
+
def __init__(self, _current_metadata: typing.Optional[str] = None):
|
1772
1772
|
...
|
1773
1773
|
@property
|
1774
1774
|
def flows(self) -> typing.List[metaflow.client.core.Flow]:
|