ob-metaflow-stubs 6.0.3.117__py2.py3-none-any.whl → 6.0.3.119__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +318 -2738
- metaflow-stubs/cards.pyi +19 -473
- metaflow-stubs/cli.pyi +17 -81
- metaflow-stubs/client/__init__.pyi +19 -1113
- metaflow-stubs/client/core.pyi +18 -158
- metaflow-stubs/client/filecache.pyi +8 -12
- metaflow-stubs/clone_util.pyi +6 -26
- metaflow-stubs/events.pyi +7 -6
- metaflow-stubs/exception.pyi +8 -6
- metaflow-stubs/flowspec.pyi +20 -104
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +16 -564
- metaflow-stubs/info_file.pyi +6 -5
- metaflow-stubs/metadata_provider/__init__.pyi +16 -0
- metaflow-stubs/metadata_provider/heartbeat.pyi +34 -0
- metaflow-stubs/{metadata → metadata_provider}/metadata.pyi +10 -22
- metaflow-stubs/metadata_provider/util.pyi +19 -0
- metaflow-stubs/metaflow_config.pyi +8 -13
- metaflow-stubs/metaflow_current.pyi +45 -44
- metaflow-stubs/mf_extensions/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +129 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +26 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +156 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +25 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +111 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +188 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +30 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +28 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +19 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +115 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +40 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +71 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +56 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +67 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +49 -0
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +130 -0
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +42 -0
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +17 -0
- metaflow-stubs/mflog/__init__.pyi +6 -0
- metaflow-stubs/mflog/mflog.pyi +52 -5
- metaflow-stubs/multicore_utils.pyi +6 -5
- metaflow-stubs/parameters.pyi +13 -23
- metaflow-stubs/plugins/__init__.pyi +52 -165
- metaflow-stubs/plugins/airflow/__init__.pyi +9 -5
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +7 -6
- metaflow-stubs/plugins/airflow/exception.pyi +7 -11
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +10 -97
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +9 -30
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +9 -40
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +9 -40
- metaflow-stubs/plugins/argo/__init__.pyi +12 -5
- metaflow-stubs/plugins/argo/argo_client.pyi +8 -26
- metaflow-stubs/plugins/argo/argo_events.pyi +7 -11
- metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -129
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +22 -460
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +11 -403
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +63 -448
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +165 -0
- metaflow-stubs/plugins/aws/__init__.pyi +11 -5
- metaflow-stubs/plugins/aws/aws_client.pyi +6 -5
- metaflow-stubs/plugins/aws/aws_utils.pyi +6 -11
- metaflow-stubs/plugins/aws/batch/__init__.pyi +9 -5
- metaflow-stubs/plugins/aws/batch/batch.pyi +10 -55
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +7 -11
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +15 -140
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +7 -5
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +10 -21
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +9 -5
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +6 -5
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +6 -5
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +7 -5
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +11 -65
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +6 -5
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +52 -292
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +127 -0
- metaflow-stubs/plugins/azure/__init__.pyi +12 -7
- metaflow-stubs/plugins/azure/azure_credential.pyi +6 -5
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +7 -11
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +11 -24
- metaflow-stubs/plugins/azure/azure_utils.pyi +11 -29
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +8 -23
- metaflow-stubs/plugins/azure/includefile_support.pyi +7 -17
- metaflow-stubs/plugins/cards/__init__.pyi +15 -5
- metaflow-stubs/plugins/cards/card_cli.pyi +22 -491
- metaflow-stubs/plugins/cards/card_client.pyi +14 -76
- metaflow-stubs/plugins/cards/card_creator.pyi +7 -10
- metaflow-stubs/plugins/cards/card_datastore.pyi +10 -18
- metaflow-stubs/plugins/cards/card_decorator.pyi +10 -126
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +13 -81
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +13 -96
- metaflow-stubs/plugins/cards/card_modules/card.pyi +6 -5
- metaflow-stubs/plugins/cards/card_modules/components.pyi +24 -107
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +6 -5
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +6 -12
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +11 -88
- metaflow-stubs/plugins/cards/card_resolver.pyi +6 -49
- metaflow-stubs/plugins/cards/component_serializer.pyi +13 -63
- metaflow-stubs/plugins/cards/exception.pyi +7 -11
- metaflow-stubs/plugins/catch_decorator.pyi +10 -30
- metaflow-stubs/plugins/datatools/__init__.pyi +13 -392
- metaflow-stubs/plugins/datatools/local.pyi +7 -11
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +19 -653
- metaflow-stubs/plugins/datatools/s3/s3.pyi +15 -263
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +7 -10
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +6 -11
- metaflow-stubs/plugins/debug_logger.pyi +7 -5
- metaflow-stubs/plugins/debug_monitor.pyi +7 -5
- metaflow-stubs/plugins/environment_decorator.pyi +7 -5
- metaflow-stubs/plugins/events_decorator.pyi +8 -14
- metaflow-stubs/plugins/frameworks/__init__.pyi +7 -5
- metaflow-stubs/plugins/frameworks/pytorch.pyi +8 -45
- metaflow-stubs/plugins/gcp/__init__.pyi +11 -7
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +11 -24
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +7 -11
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +6 -5
- metaflow-stubs/plugins/gcp/gs_utils.pyi +8 -20
- metaflow-stubs/plugins/gcp/includefile_support.pyi +7 -17
- metaflow-stubs/plugins/kubernetes/__init__.pyi +12 -5
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +6 -10
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +10 -38
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +16 -155
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +7 -11
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +19 -142
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +7 -11
- metaflow-stubs/plugins/logs_cli.pyi +10 -9
- metaflow-stubs/plugins/package_cli.pyi +7 -5
- metaflow-stubs/plugins/parallel_decorator.pyi +11 -59
- metaflow-stubs/plugins/perimeters.pyi +6 -11
- metaflow-stubs/plugins/project_decorator.pyi +8 -14
- metaflow-stubs/plugins/pypi/__init__.pyi +12 -11
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +8 -27
- metaflow-stubs/plugins/pypi/conda_environment.pyi +14 -20
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +7 -5
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +7 -39
- metaflow-stubs/plugins/pypi/utils.pyi +7 -11
- metaflow-stubs/plugins/resources_decorator.pyi +7 -5
- metaflow-stubs/plugins/retry_decorator.pyi +7 -11
- metaflow-stubs/plugins/secrets/__init__.pyi +9 -5
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +8 -13
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +7 -11
- metaflow-stubs/plugins/storage_executor.pyi +6 -11
- metaflow-stubs/plugins/tag_cli.pyi +14 -396
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +9 -34
- metaflow-stubs/plugins/timeout_decorator.pyi +8 -12
- metaflow-stubs/procpoll.pyi +7 -5
- metaflow-stubs/profilers/__init__.pyi +7 -11
- metaflow-stubs/pylint_wrapper.pyi +7 -11
- metaflow-stubs/runner/__init__.pyi +13 -5
- metaflow-stubs/runner/deployer.pyi +99 -223
- metaflow-stubs/runner/deployer_impl.pyi +87 -0
- metaflow-stubs/runner/metaflow_runner.pyi +24 -508
- metaflow-stubs/runner/nbdeploy.pyi +16 -60
- metaflow-stubs/runner/nbrun.pyi +11 -148
- metaflow-stubs/runner/subprocess_manager.pyi +9 -10
- metaflow-stubs/runner/utils.pyi +44 -9
- metaflow-stubs/system/__init__.pyi +9 -107
- metaflow-stubs/system/system_logger.pyi +7 -16
- metaflow-stubs/system/system_monitor.pyi +6 -16
- metaflow-stubs/tagging_util.pyi +6 -10
- metaflow-stubs/tuple_util.pyi +6 -5
- {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.119.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.119.dist-info/RECORD +174 -0
- metaflow-stubs/metadata/util.pyi +0 -18
- ob_metaflow_stubs-6.0.3.117.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.119.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.119.dist-info}/top_level.txt +0 -0
@@ -1,12 +1,13 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.848392 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
|
10
|
+
|
10
11
|
class GcpDefaultClientProvider(object, metaclass=type):
|
11
12
|
@staticmethod
|
12
13
|
def get_gs_storage_client(*args, **kwargs):
|
@@ -1,27 +1,15 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.848750 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
-
import typing
|
10
|
-
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.exception
|
12
9
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
def __str__(self):
|
17
|
-
...
|
18
|
-
...
|
19
|
-
|
20
|
-
class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
|
21
|
-
...
|
22
|
-
|
23
|
-
class MetaflowGSPackageError(metaflow.exception.MetaflowException, metaclass=type):
|
24
|
-
...
|
10
|
+
from ...exception import MetaflowException as MetaflowException
|
11
|
+
from ...exception import MetaflowInternalError as MetaflowInternalError
|
12
|
+
from .gs_exceptions import MetaflowGSPackageError as MetaflowGSPackageError
|
25
13
|
|
26
14
|
def parse_gs_full_path(gs_uri):
|
27
15
|
...
|
@@ -1,24 +1,14 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.849198 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
-
import typing
|
10
|
-
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.exception
|
12
9
|
|
13
|
-
|
14
|
-
|
15
|
-
...
|
16
|
-
def __str__(self):
|
17
|
-
...
|
18
|
-
...
|
19
|
-
|
20
|
-
class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
|
21
|
-
...
|
10
|
+
from ...exception import MetaflowException as MetaflowException
|
11
|
+
from ...exception import MetaflowInternalError as MetaflowInternalError
|
22
12
|
|
23
13
|
class GS(object, metaclass=type):
|
24
14
|
@classmethod
|
@@ -1,9 +1,16 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.827161 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
|
10
|
+
from . import kubernetes_client as kubernetes_client
|
11
|
+
from . import kubernetes as kubernetes
|
12
|
+
from . import kubernetes_decorator as kubernetes_decorator
|
13
|
+
from . import kubernetes_jobsets as kubernetes_jobsets
|
14
|
+
from . import kube_utils as kube_utils
|
15
|
+
from . import kubernetes_cli as kubernetes_cli
|
16
|
+
|
@@ -1,17 +1,13 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.842914 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
-
import typing
|
10
|
-
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.exception
|
12
9
|
|
13
|
-
|
14
|
-
...
|
10
|
+
from ...exception import CommandException as CommandException
|
15
11
|
|
16
12
|
def parse_cli_options(flow_name, run_id, user, my_runs, echo):
|
17
13
|
...
|
@@ -1,24 +1,20 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.840842 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
+
import metaflow
|
9
10
|
import typing
|
10
11
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.metaflow_current
|
12
12
|
import metaflow.exception
|
13
13
|
|
14
|
-
current
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
...
|
19
|
-
def __str__(self):
|
20
|
-
...
|
21
|
-
...
|
14
|
+
from ...metaflow_current import current as current
|
15
|
+
from ...exception import MetaflowException as MetaflowException
|
16
|
+
from ...mf_extensions.outerbounds.remote_config import init_config as init_config
|
17
|
+
from .kubernetes_client import KubernetesClient as KubernetesClient
|
22
18
|
|
23
19
|
ARGO_EVENTS_EVENT: None
|
24
20
|
|
@@ -88,32 +84,8 @@ UBF_CONTROL: str
|
|
88
84
|
|
89
85
|
UBF_TASK: str
|
90
86
|
|
91
|
-
def init_config() -> typing.Dict[str, str]:
|
92
|
-
"""
|
93
|
-
OSS Metaflow reads the config file on every step initialization. This is because OSS assumes config files change
|
94
|
-
relatively infrequently. We want to avoid config values changing between flow steps. Our solution to prevent this
|
95
|
-
is to read a config once and cache it on an environment variable. Environment variables carry over between steps
|
96
|
-
because steps are executed in subprocesses (local) or environments which expect environment variables to be set.
|
97
|
-
"""
|
98
|
-
...
|
99
|
-
|
100
87
|
BASH_SAVE_LOGS: str
|
101
88
|
|
102
|
-
class KubernetesClient(object, metaclass=type):
|
103
|
-
def __init__(self):
|
104
|
-
...
|
105
|
-
def get(self):
|
106
|
-
...
|
107
|
-
def list(self, flow_name, run_id, user):
|
108
|
-
...
|
109
|
-
def kill_pods(self, flow_name, run_id, user, echo):
|
110
|
-
...
|
111
|
-
def job(self, **kwargs):
|
112
|
-
...
|
113
|
-
def jobset(self, **kwargs):
|
114
|
-
...
|
115
|
-
...
|
116
|
-
|
117
89
|
LOGS_DIR: str
|
118
90
|
|
119
91
|
STDOUT_FILE: str
|
@@ -1,56 +1,26 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.843197 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
-
import typing
|
10
|
-
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.decorators
|
12
|
-
import metaflow.exception
|
13
|
-
import metaflow._vendor.click.types
|
14
9
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
...
|
27
|
-
def job(self, **kwargs):
|
28
|
-
...
|
29
|
-
def jobset(self, **kwargs):
|
30
|
-
...
|
31
|
-
...
|
32
|
-
|
33
|
-
class JSONTypeClass(metaflow._vendor.click.types.ParamType, metaclass=type):
|
34
|
-
def convert(self, value, param, ctx):
|
35
|
-
...
|
36
|
-
def __str__(self):
|
37
|
-
...
|
38
|
-
def __repr__(self):
|
39
|
-
...
|
40
|
-
...
|
10
|
+
from .kube_utils import parse_cli_options as parse_cli_options
|
11
|
+
from .kubernetes_client import KubernetesClient as KubernetesClient
|
12
|
+
from ...parameters import JSONTypeClass as JSONTypeClass
|
13
|
+
from ..._vendor import click as click
|
14
|
+
from ...exception import MetaflowException as MetaflowException
|
15
|
+
from ...metadata_provider.util import sync_local_metadata_from_datastore as sync_local_metadata_from_datastore
|
16
|
+
from .kubernetes import Kubernetes as Kubernetes
|
17
|
+
from .kubernetes import KubernetesException as KubernetesException
|
18
|
+
from .kubernetes import KubernetesKilledException as KubernetesKilledException
|
19
|
+
from .kubernetes import parse_kube_keyvalue_list as parse_kube_keyvalue_list
|
20
|
+
from .kubernetes_decorator import KubernetesDecorator as KubernetesDecorator
|
41
21
|
|
42
22
|
METAFLOW_EXIT_DISALLOW_RETRY: int
|
43
23
|
|
44
|
-
class MetaflowException(Exception, metaclass=type):
|
45
|
-
def __init__(self, msg = "", lineno = None):
|
46
|
-
...
|
47
|
-
def __str__(self):
|
48
|
-
...
|
49
|
-
...
|
50
|
-
|
51
|
-
def sync_local_metadata_from_datastore(metadata_local_dir, task_ds):
|
52
|
-
...
|
53
|
-
|
54
24
|
DATASTORE_LOCAL_DIR: str
|
55
25
|
|
56
26
|
KUBERNETES_LABELS: str
|
@@ -61,112 +31,3 @@ UBF_CONTROL: str
|
|
61
31
|
|
62
32
|
UBF_TASK: str
|
63
33
|
|
64
|
-
class Kubernetes(object, metaclass=type):
|
65
|
-
def __init__(self, datastore, metadata, environment):
|
66
|
-
...
|
67
|
-
def launch_job(self, **kwargs):
|
68
|
-
...
|
69
|
-
def create_jobset(self, flow_name, run_id, step_name, task_id, attempt, user, code_package_sha, code_package_url, code_package_ds, docker_image, docker_image_pull_policy, step_cli = None, service_account = None, secrets = None, node_selector = None, namespace = None, cpu = None, gpu = None, gpu_vendor = None, disk = None, memory = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, run_time_limit = None, env = None, persistent_volume_claims = None, tolerations = None, labels = None, shared_memory = None, port = None, num_parallel = None):
|
70
|
-
...
|
71
|
-
def create_job_object(self, flow_name, run_id, step_name, task_id, attempt, user, code_package_sha, code_package_url, code_package_ds, step_cli, docker_image, docker_image_pull_policy, service_account = None, secrets = None, node_selector = None, namespace = None, cpu = None, gpu = None, gpu_vendor = None, disk = None, memory = None, use_tmpfs = None, tmpfs_tempdir = None, tmpfs_size = None, tmpfs_path = None, run_time_limit = None, env = None, persistent_volume_claims = None, tolerations = None, labels = None, shared_memory = None, port = None, name_pattern = None):
|
72
|
-
...
|
73
|
-
def create_k8sjob(self, job):
|
74
|
-
...
|
75
|
-
def wait(self, stdout_location, stderr_location, echo = None):
|
76
|
-
...
|
77
|
-
...
|
78
|
-
|
79
|
-
class KubernetesException(metaflow.exception.MetaflowException, metaclass=type):
|
80
|
-
...
|
81
|
-
|
82
|
-
class KubernetesKilledException(metaflow.exception.MetaflowException, metaclass=type):
|
83
|
-
...
|
84
|
-
|
85
|
-
def parse_kube_keyvalue_list(items: typing.List[str], requires_both: bool = True):
|
86
|
-
...
|
87
|
-
|
88
|
-
class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
89
|
-
"""
|
90
|
-
Specifies that this step should execute on Kubernetes.
|
91
|
-
|
92
|
-
Parameters
|
93
|
-
----------
|
94
|
-
cpu : int, default 1
|
95
|
-
Number of CPUs required for this step. If `@resources` is
|
96
|
-
also present, the maximum value from all decorators is used.
|
97
|
-
memory : int, default 4096
|
98
|
-
Memory size (in MB) required for this step. If
|
99
|
-
`@resources` is also present, the maximum value from all decorators is
|
100
|
-
used.
|
101
|
-
disk : int, default 10240
|
102
|
-
Disk size (in MB) required for this step. If
|
103
|
-
`@resources` is also present, the maximum value from all decorators is
|
104
|
-
used.
|
105
|
-
image : str, optional, default None
|
106
|
-
Docker image to use when launching on Kubernetes. If not specified, and
|
107
|
-
METAFLOW_KUBERNETES_CONTAINER_IMAGE is specified, that image is used. If
|
108
|
-
not, a default Docker image mapping to the current version of Python is used.
|
109
|
-
image_pull_policy: str, default KUBERNETES_IMAGE_PULL_POLICY
|
110
|
-
If given, the imagePullPolicy to be applied to the Docker image of the step.
|
111
|
-
service_account : str, default METAFLOW_KUBERNETES_SERVICE_ACCOUNT
|
112
|
-
Kubernetes service account to use when launching pod in Kubernetes.
|
113
|
-
secrets : List[str], optional, default None
|
114
|
-
Kubernetes secrets to use when launching pod in Kubernetes. These
|
115
|
-
secrets are in addition to the ones defined in `METAFLOW_KUBERNETES_SECRETS`
|
116
|
-
in Metaflow configuration.
|
117
|
-
node_selector: Union[Dict[str,str], str], optional, default None
|
118
|
-
Kubernetes node selector(s) to apply to the pod running the task.
|
119
|
-
Can be passed in as a comma separated string of values e.g. "kubernetes.io/os=linux,kubernetes.io/arch=amd64"
|
120
|
-
or as a dictionary {"kubernetes.io/os": "linux", "kubernetes.io/arch": "amd64"}
|
121
|
-
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
122
|
-
Kubernetes namespace to use when launching pod in Kubernetes.
|
123
|
-
gpu : int, optional, default None
|
124
|
-
Number of GPUs required for this step. A value of zero implies that
|
125
|
-
the scheduled node should not have GPUs.
|
126
|
-
gpu_vendor : str, default KUBERNETES_GPU_VENDOR
|
127
|
-
The vendor of the GPUs to be used for this step.
|
128
|
-
tolerations : List[str], default []
|
129
|
-
The default is extracted from METAFLOW_KUBERNETES_TOLERATIONS.
|
130
|
-
Kubernetes tolerations to use when launching pod in Kubernetes.
|
131
|
-
use_tmpfs : bool, default False
|
132
|
-
This enables an explicit tmpfs mount for this step.
|
133
|
-
tmpfs_tempdir : bool, default True
|
134
|
-
sets METAFLOW_TEMPDIR to tmpfs_path if set for this step.
|
135
|
-
tmpfs_size : int, optional, default: None
|
136
|
-
The value for the size (in MiB) of the tmpfs mount for this step.
|
137
|
-
This parameter maps to the `--tmpfs` option in Docker. Defaults to 50% of the
|
138
|
-
memory allocated for this step.
|
139
|
-
tmpfs_path : str, optional, default /metaflow_temp
|
140
|
-
Path to tmpfs mount for this step.
|
141
|
-
persistent_volume_claims : Dict[str, str], optional, default None
|
142
|
-
A map (dictionary) of persistent volumes to be mounted to the pod for this step. The map is from persistent
|
143
|
-
volumes to the path to which the volume is to be mounted, e.g., `{'pvc-name': '/path/to/mount/on'}`.
|
144
|
-
shared_memory: int, optional
|
145
|
-
Shared memory size (in MiB) required for this step
|
146
|
-
port: int, optional
|
147
|
-
Port number to specify in the Kubernetes job object
|
148
|
-
compute_pool : str, optional, default None
|
149
|
-
Compute pool to be used for for this step.
|
150
|
-
If not specified, any accessible compute pool within the perimeter is used.
|
151
|
-
hostname_resolution_timeout: int, default 10 * 60
|
152
|
-
Timeout in seconds for the workers tasks in the gang scheduled cluster to resolve the hostname of control task.
|
153
|
-
Only applicable when @parallel is used.
|
154
|
-
"""
|
155
|
-
def __init__(self, attributes = None, statically_defined = False):
|
156
|
-
...
|
157
|
-
def step_init(self, flow, graph, step, decos, environment, flow_datastore, logger):
|
158
|
-
...
|
159
|
-
def package_init(self, flow, step_name, environment):
|
160
|
-
...
|
161
|
-
def runtime_init(self, flow, graph, package, run_id):
|
162
|
-
...
|
163
|
-
def runtime_task_created(self, task_datastore, task_id, split_index, input_paths, is_cloned, ubf_context):
|
164
|
-
...
|
165
|
-
def runtime_step_cli(self, cli_args, retry_count, max_user_code_retries, ubf_context):
|
166
|
-
...
|
167
|
-
def task_pre_step(self, step_name, task_datastore, metadata, run_id, task_id, flow, graph, retry_count, max_retries, ubf_context, inputs):
|
168
|
-
...
|
169
|
-
def task_finished(self, step_name, flow, graph, is_task_ok, retry_count, max_retries):
|
170
|
-
...
|
171
|
-
...
|
172
|
-
|
@@ -1,21 +1,17 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.796967 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
+
import metaflow
|
9
10
|
import typing
|
10
11
|
if typing.TYPE_CHECKING:
|
11
12
|
import metaflow.exception
|
12
13
|
|
13
|
-
|
14
|
-
def __init__(self, msg = "", lineno = None):
|
15
|
-
...
|
16
|
-
def __str__(self):
|
17
|
-
...
|
18
|
-
...
|
14
|
+
from ...exception import MetaflowException as MetaflowException
|
19
15
|
|
20
16
|
KUBERNETES_NAMESPACE: str
|
21
17
|
|
@@ -1,50 +1,26 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.841522 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
+
import metaflow
|
9
10
|
import typing
|
10
11
|
if typing.TYPE_CHECKING:
|
11
12
|
import metaflow.decorators
|
12
|
-
import metaflow.metaflow_current
|
13
|
-
import metaflow.exception
|
14
13
|
|
15
|
-
current
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
"""
|
26
|
-
MetaDatum(field, value, type, tags)
|
27
|
-
"""
|
28
|
-
@staticmethod
|
29
|
-
def __new__(_cls, field, value, type, tags):
|
30
|
-
"""
|
31
|
-
Create new instance of MetaDatum(field, value, type, tags)
|
32
|
-
"""
|
33
|
-
...
|
34
|
-
def __repr__(self):
|
35
|
-
"""
|
36
|
-
Return a nicely formatted representation string
|
37
|
-
"""
|
38
|
-
...
|
39
|
-
def __getnewargs__(self):
|
40
|
-
"""
|
41
|
-
Return self as a plain tuple. Used by copy and pickle.
|
42
|
-
"""
|
43
|
-
...
|
44
|
-
...
|
45
|
-
|
46
|
-
def sync_local_metadata_to_datastore(metadata_local_dir, task_ds):
|
47
|
-
...
|
14
|
+
from ...metaflow_current import current as current
|
15
|
+
from ...exception import MetaflowException as MetaflowException
|
16
|
+
from ...metadata_provider.metadata import MetaDatum as MetaDatum
|
17
|
+
from ...metadata_provider.util import sync_local_metadata_to_datastore as sync_local_metadata_to_datastore
|
18
|
+
from ..resources_decorator import ResourcesDecorator as ResourcesDecorator
|
19
|
+
from ..timeout_decorator import get_run_time_limit_for_task as get_run_time_limit_for_task
|
20
|
+
from ..aws.aws_utils import get_docker_registry as get_docker_registry
|
21
|
+
from ..aws.aws_utils import get_ec2_instance_metadata as get_ec2_instance_metadata
|
22
|
+
from .kubernetes import KubernetesException as KubernetesException
|
23
|
+
from .kubernetes import parse_kube_keyvalue_list as parse_kube_keyvalue_list
|
48
24
|
|
49
25
|
DATASTORE_LOCAL_DIR: str
|
50
26
|
|
@@ -78,108 +54,8 @@ KUBERNETES_SHARED_MEMORY: None
|
|
78
54
|
|
79
55
|
KUBERNETES_TOLERATIONS: str
|
80
56
|
|
81
|
-
class ResourcesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
82
|
-
"""
|
83
|
-
Specifies the resources needed when executing this step.
|
84
|
-
|
85
|
-
Use `@resources` to specify the resource requirements
|
86
|
-
independently of the specific compute layer (`@batch`, `@kubernetes`).
|
87
|
-
|
88
|
-
You can choose the compute layer on the command line by executing e.g.
|
89
|
-
```
|
90
|
-
python myflow.py run --with batch
|
91
|
-
```
|
92
|
-
or
|
93
|
-
```
|
94
|
-
python myflow.py run --with kubernetes
|
95
|
-
```
|
96
|
-
which executes the flow on the desired system using the
|
97
|
-
requirements specified in `@resources`.
|
98
|
-
|
99
|
-
Parameters
|
100
|
-
----------
|
101
|
-
cpu : int, default 1
|
102
|
-
Number of CPUs required for this step.
|
103
|
-
gpu : int, optional, default None
|
104
|
-
Number of GPUs required for this step.
|
105
|
-
disk : int, optional, default None
|
106
|
-
Disk size (in MB) required for this step. Only applies on Kubernetes.
|
107
|
-
memory : int, default 4096
|
108
|
-
Memory size (in MB) required for this step.
|
109
|
-
shared_memory : int, optional, default None
|
110
|
-
The value for the size (in MiB) of the /dev/shm volume for this step.
|
111
|
-
This parameter maps to the `--shm-size` option in Docker.
|
112
|
-
"""
|
113
|
-
...
|
114
|
-
|
115
|
-
def get_run_time_limit_for_task(step_decos):
|
116
|
-
...
|
117
|
-
|
118
57
|
UBF_CONTROL: str
|
119
58
|
|
120
|
-
def get_docker_registry(image_uri):
|
121
|
-
"""
|
122
|
-
Explanation:
|
123
|
-
(.+?(?:[:.].+?)\/)? - [GROUP 0] REGISTRY
|
124
|
-
.+? - A registry must start with at least one character
|
125
|
-
(?:[:.].+?)\/ - A registry must have ":" or "." and end with "/"
|
126
|
-
? - Make a registry optional
|
127
|
-
(.*?) - [GROUP 1] REPOSITORY
|
128
|
-
.*? - Get repository name until separator
|
129
|
-
(?:[@:])? - SEPARATOR
|
130
|
-
?: - Don't capture separator
|
131
|
-
[@:] - The separator must be either "@" or ":"
|
132
|
-
? - The separator is optional
|
133
|
-
((?<=[@:]).*)? - [GROUP 2] TAG / DIGEST
|
134
|
-
(?<=[@:]) - A tag / digest must be preceded by "@" or ":"
|
135
|
-
.* - Capture rest of tag / digest
|
136
|
-
? - A tag / digest is optional
|
137
|
-
Examples:
|
138
|
-
image
|
139
|
-
- None
|
140
|
-
- image
|
141
|
-
- None
|
142
|
-
example/image
|
143
|
-
- None
|
144
|
-
- example/image
|
145
|
-
- None
|
146
|
-
example/image:tag
|
147
|
-
- None
|
148
|
-
- example/image
|
149
|
-
- tag
|
150
|
-
example.domain.com/example/image:tag
|
151
|
-
- example.domain.com/
|
152
|
-
- example/image
|
153
|
-
- tag
|
154
|
-
123.123.123.123:123/example/image:tag
|
155
|
-
- 123.123.123.123:123/
|
156
|
-
- example/image
|
157
|
-
- tag
|
158
|
-
example.domain.com/example/image@sha256:45b23dee0
|
159
|
-
- example.domain.com/
|
160
|
-
- example/image
|
161
|
-
- sha256:45b23dee0
|
162
|
-
"""
|
163
|
-
...
|
164
|
-
|
165
|
-
def get_ec2_instance_metadata():
|
166
|
-
"""
|
167
|
-
Fetches the EC2 instance metadata through AWS instance metadata service
|
168
|
-
|
169
|
-
Returns either an empty dictionary, or one with the keys
|
170
|
-
- ec2-instance-id
|
171
|
-
- ec2-instance-type
|
172
|
-
- ec2-region
|
173
|
-
- ec2-availability-zone
|
174
|
-
"""
|
175
|
-
...
|
176
|
-
|
177
|
-
class KubernetesException(metaflow.exception.MetaflowException, metaclass=type):
|
178
|
-
...
|
179
|
-
|
180
|
-
def parse_kube_keyvalue_list(items: typing.List[str], requires_both: bool = True):
|
181
|
-
...
|
182
|
-
|
183
59
|
MAX_MEMORY_PER_TASK: None
|
184
60
|
|
185
61
|
MAX_CPU_PER_TASK: None
|
@@ -215,8 +91,9 @@ class KubernetesDecorator(metaflow.decorators.StepDecorator, metaclass=type):
|
|
215
91
|
in Metaflow configuration.
|
216
92
|
node_selector: Union[Dict[str,str], str], optional, default None
|
217
93
|
Kubernetes node selector(s) to apply to the pod running the task.
|
218
|
-
Can be passed in as a comma separated string of values e.g.
|
219
|
-
|
94
|
+
Can be passed in as a comma separated string of values e.g.
|
95
|
+
'kubernetes.io/os=linux,kubernetes.io/arch=amd64' or as a dictionary
|
96
|
+
{'kubernetes.io/os': 'linux', 'kubernetes.io/arch': 'amd64'}
|
220
97
|
namespace : str, default METAFLOW_KUBERNETES_NAMESPACE
|
221
98
|
Kubernetes namespace to use when launching pod in Kubernetes.
|
222
99
|
gpu : int, optional, default None
|
@@ -1,21 +1,17 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.842732 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
+
import metaflow
|
9
10
|
import typing
|
10
11
|
if typing.TYPE_CHECKING:
|
11
12
|
import metaflow.exception
|
12
13
|
|
13
|
-
|
14
|
-
def __init__(self, msg = "", lineno = None):
|
15
|
-
...
|
16
|
-
def __str__(self):
|
17
|
-
...
|
18
|
-
...
|
14
|
+
from ...exception import MetaflowException as MetaflowException
|
19
15
|
|
20
16
|
KUBERNETES_JOBSET_GROUP: str
|
21
17
|
|