ob-metaflow-stubs 6.0.3.117__py2.py3-none-any.whl → 6.0.3.119__py2.py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- metaflow-stubs/__init__.pyi +318 -2738
- metaflow-stubs/cards.pyi +19 -473
- metaflow-stubs/cli.pyi +17 -81
- metaflow-stubs/client/__init__.pyi +19 -1113
- metaflow-stubs/client/core.pyi +18 -158
- metaflow-stubs/client/filecache.pyi +8 -12
- metaflow-stubs/clone_util.pyi +6 -26
- metaflow-stubs/events.pyi +7 -6
- metaflow-stubs/exception.pyi +8 -6
- metaflow-stubs/flowspec.pyi +20 -104
- metaflow-stubs/generated_for.txt +1 -1
- metaflow-stubs/includefile.pyi +16 -564
- metaflow-stubs/info_file.pyi +6 -5
- metaflow-stubs/metadata_provider/__init__.pyi +16 -0
- metaflow-stubs/metadata_provider/heartbeat.pyi +34 -0
- metaflow-stubs/{metadata → metadata_provider}/metadata.pyi +10 -22
- metaflow-stubs/metadata_provider/util.pyi +19 -0
- metaflow-stubs/metaflow_config.pyi +8 -13
- metaflow-stubs/metaflow_current.pyi +45 -44
- metaflow-stubs/mf_extensions/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +129 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +26 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +156 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +25 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +111 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +188 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +30 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +28 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +19 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +115 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +40 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +71 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +56 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +67 -0
- metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +49 -0
- metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +130 -0
- metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +42 -0
- metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +6 -0
- metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +17 -0
- metaflow-stubs/mflog/__init__.pyi +6 -0
- metaflow-stubs/mflog/mflog.pyi +52 -5
- metaflow-stubs/multicore_utils.pyi +6 -5
- metaflow-stubs/parameters.pyi +13 -23
- metaflow-stubs/plugins/__init__.pyi +52 -165
- metaflow-stubs/plugins/airflow/__init__.pyi +9 -5
- metaflow-stubs/plugins/airflow/airflow_utils.pyi +7 -6
- metaflow-stubs/plugins/airflow/exception.pyi +7 -11
- metaflow-stubs/plugins/airflow/sensors/__init__.pyi +10 -97
- metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +9 -30
- metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +9 -40
- metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +9 -40
- metaflow-stubs/plugins/argo/__init__.pyi +12 -5
- metaflow-stubs/plugins/argo/argo_client.pyi +8 -26
- metaflow-stubs/plugins/argo/argo_events.pyi +7 -11
- metaflow-stubs/plugins/argo/argo_workflows.pyi +17 -129
- metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +22 -460
- metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +11 -403
- metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +63 -448
- metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +165 -0
- metaflow-stubs/plugins/aws/__init__.pyi +11 -5
- metaflow-stubs/plugins/aws/aws_client.pyi +6 -5
- metaflow-stubs/plugins/aws/aws_utils.pyi +6 -11
- metaflow-stubs/plugins/aws/batch/__init__.pyi +9 -5
- metaflow-stubs/plugins/aws/batch/batch.pyi +10 -55
- metaflow-stubs/plugins/aws/batch/batch_client.pyi +7 -11
- metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +15 -140
- metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +7 -5
- metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +10 -21
- metaflow-stubs/plugins/aws/step_functions/__init__.pyi +9 -5
- metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +6 -5
- metaflow-stubs/plugins/aws/step_functions/production_token.pyi +6 -5
- metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +7 -5
- metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +11 -65
- metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +6 -5
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +52 -292
- metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +127 -0
- metaflow-stubs/plugins/azure/__init__.pyi +12 -7
- metaflow-stubs/plugins/azure/azure_credential.pyi +6 -5
- metaflow-stubs/plugins/azure/azure_exceptions.pyi +7 -11
- metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +11 -24
- metaflow-stubs/plugins/azure/azure_utils.pyi +11 -29
- metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +8 -23
- metaflow-stubs/plugins/azure/includefile_support.pyi +7 -17
- metaflow-stubs/plugins/cards/__init__.pyi +15 -5
- metaflow-stubs/plugins/cards/card_cli.pyi +22 -491
- metaflow-stubs/plugins/cards/card_client.pyi +14 -76
- metaflow-stubs/plugins/cards/card_creator.pyi +7 -10
- metaflow-stubs/plugins/cards/card_datastore.pyi +10 -18
- metaflow-stubs/plugins/cards/card_decorator.pyi +10 -126
- metaflow-stubs/plugins/cards/card_modules/__init__.pyi +13 -81
- metaflow-stubs/plugins/cards/card_modules/basic.pyi +13 -96
- metaflow-stubs/plugins/cards/card_modules/card.pyi +6 -5
- metaflow-stubs/plugins/cards/card_modules/components.pyi +24 -107
- metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +6 -5
- metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +6 -12
- metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +11 -88
- metaflow-stubs/plugins/cards/card_resolver.pyi +6 -49
- metaflow-stubs/plugins/cards/component_serializer.pyi +13 -63
- metaflow-stubs/plugins/cards/exception.pyi +7 -11
- metaflow-stubs/plugins/catch_decorator.pyi +10 -30
- metaflow-stubs/plugins/datatools/__init__.pyi +13 -392
- metaflow-stubs/plugins/datatools/local.pyi +7 -11
- metaflow-stubs/plugins/datatools/s3/__init__.pyi +19 -653
- metaflow-stubs/plugins/datatools/s3/s3.pyi +15 -263
- metaflow-stubs/plugins/datatools/s3/s3tail.pyi +7 -10
- metaflow-stubs/plugins/datatools/s3/s3util.pyi +6 -11
- metaflow-stubs/plugins/debug_logger.pyi +7 -5
- metaflow-stubs/plugins/debug_monitor.pyi +7 -5
- metaflow-stubs/plugins/environment_decorator.pyi +7 -5
- metaflow-stubs/plugins/events_decorator.pyi +8 -14
- metaflow-stubs/plugins/frameworks/__init__.pyi +7 -5
- metaflow-stubs/plugins/frameworks/pytorch.pyi +8 -45
- metaflow-stubs/plugins/gcp/__init__.pyi +11 -7
- metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +11 -24
- metaflow-stubs/plugins/gcp/gs_exceptions.pyi +7 -11
- metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +6 -5
- metaflow-stubs/plugins/gcp/gs_utils.pyi +8 -20
- metaflow-stubs/plugins/gcp/includefile_support.pyi +7 -17
- metaflow-stubs/plugins/kubernetes/__init__.pyi +12 -5
- metaflow-stubs/plugins/kubernetes/kube_utils.pyi +6 -10
- metaflow-stubs/plugins/kubernetes/kubernetes.pyi +10 -38
- metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +16 -155
- metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +7 -11
- metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +19 -142
- metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +7 -11
- metaflow-stubs/plugins/logs_cli.pyi +10 -9
- metaflow-stubs/plugins/package_cli.pyi +7 -5
- metaflow-stubs/plugins/parallel_decorator.pyi +11 -59
- metaflow-stubs/plugins/perimeters.pyi +6 -11
- metaflow-stubs/plugins/project_decorator.pyi +8 -14
- metaflow-stubs/plugins/pypi/__init__.pyi +12 -11
- metaflow-stubs/plugins/pypi/conda_decorator.pyi +8 -27
- metaflow-stubs/plugins/pypi/conda_environment.pyi +14 -20
- metaflow-stubs/plugins/pypi/pypi_decorator.pyi +7 -5
- metaflow-stubs/plugins/pypi/pypi_environment.pyi +7 -39
- metaflow-stubs/plugins/pypi/utils.pyi +7 -11
- metaflow-stubs/plugins/resources_decorator.pyi +7 -5
- metaflow-stubs/plugins/retry_decorator.pyi +7 -11
- metaflow-stubs/plugins/secrets/__init__.pyi +9 -5
- metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +8 -13
- metaflow-stubs/plugins/secrets/secrets_decorator.pyi +7 -11
- metaflow-stubs/plugins/storage_executor.pyi +6 -11
- metaflow-stubs/plugins/tag_cli.pyi +14 -396
- metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +9 -34
- metaflow-stubs/plugins/timeout_decorator.pyi +8 -12
- metaflow-stubs/procpoll.pyi +7 -5
- metaflow-stubs/profilers/__init__.pyi +7 -11
- metaflow-stubs/pylint_wrapper.pyi +7 -11
- metaflow-stubs/runner/__init__.pyi +13 -5
- metaflow-stubs/runner/deployer.pyi +99 -223
- metaflow-stubs/runner/deployer_impl.pyi +87 -0
- metaflow-stubs/runner/metaflow_runner.pyi +24 -508
- metaflow-stubs/runner/nbdeploy.pyi +16 -60
- metaflow-stubs/runner/nbrun.pyi +11 -148
- metaflow-stubs/runner/subprocess_manager.pyi +9 -10
- metaflow-stubs/runner/utils.pyi +44 -9
- metaflow-stubs/system/__init__.pyi +9 -107
- metaflow-stubs/system/system_logger.pyi +7 -16
- metaflow-stubs/system/system_monitor.pyi +6 -16
- metaflow-stubs/tagging_util.pyi +6 -10
- metaflow-stubs/tuple_util.pyi +6 -5
- {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.119.dist-info}/METADATA +1 -1
- ob_metaflow_stubs-6.0.3.119.dist-info/RECORD +174 -0
- metaflow-stubs/metadata/util.pyi +0 -18
- ob_metaflow_stubs-6.0.3.117.dist-info/RECORD +0 -140
- {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.119.dist-info}/WHEEL +0 -0
- {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.119.dist-info}/top_level.txt +0 -0
@@ -1,31 +1,19 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.818374 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
|
+
import metaflow
|
9
10
|
import typing
|
10
11
|
if typing.TYPE_CHECKING:
|
11
|
-
import metaflow.
|
12
|
-
import metaflow.exception
|
12
|
+
import metaflow.metadata_provider.metadata
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
class MetaflowTaggingError(metaflow.exception.MetaflowException, metaclass=type):
|
18
|
-
...
|
19
|
-
|
20
|
-
def validate_tag(tag):
|
21
|
-
"""
|
22
|
-
- Tag must be either of bytes-type or unicode-type.
|
23
|
-
- If tag is of bytes-type, it must be UTF-8 decodable
|
24
|
-
- If tag is of unicode-type, it must be UTF-8 encodable
|
25
|
-
- Tag may not be empty string.
|
26
|
-
- Tag cannot be too long (500 chars)
|
27
|
-
"""
|
28
|
-
...
|
14
|
+
from ..exception import MetaflowInternalError as MetaflowInternalError
|
15
|
+
from ..exception import MetaflowTaggingError as MetaflowTaggingError
|
16
|
+
from ..tagging_util import validate_tag as validate_tag
|
29
17
|
|
30
18
|
class DataArtifact(tuple, metaclass=type):
|
31
19
|
"""
|
@@ -0,0 +1,19 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.818617 #
|
5
|
+
######################################################################################################
|
6
|
+
|
7
|
+
from __future__ import annotations
|
8
|
+
|
9
|
+
|
10
|
+
|
11
|
+
def copy_tree(src, dst, update = False):
|
12
|
+
...
|
13
|
+
|
14
|
+
def sync_local_metadata_to_datastore(metadata_local_dir, task_ds):
|
15
|
+
...
|
16
|
+
|
17
|
+
def sync_local_metadata_from_datastore(metadata_local_dir, task_ds):
|
18
|
+
...
|
19
|
+
|
@@ -1,18 +1,13 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.799422 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
|
10
|
-
|
11
|
-
def __init__(self, msg = "", lineno = None):
|
12
|
-
...
|
13
|
-
def __str__(self):
|
14
|
-
...
|
15
|
-
...
|
10
|
+
from .exception import MetaflowException as MetaflowException
|
16
11
|
|
17
12
|
DATASTORE_LOCAL_DIR: str
|
18
13
|
|
@@ -40,6 +35,8 @@ DEFAULT_SECRETS_BACKEND_TYPE: None
|
|
40
35
|
|
41
36
|
DEFAULT_SECRETS_ROLE: None
|
42
37
|
|
38
|
+
DEFAULT_FROM_DEPLOYMENT_IMPL: str
|
39
|
+
|
43
40
|
USER: None
|
44
41
|
|
45
42
|
DATASTORE_SYSROOT_LOCAL: None
|
@@ -240,8 +237,6 @@ AIRFLOW_KUBERNETES_KUBECONFIG_FILE: None
|
|
240
237
|
|
241
238
|
AIRFLOW_KUBERNETES_KUBECONFIG_CONTEXT: None
|
242
239
|
|
243
|
-
FROM_DEPLOYMENT_IMPL: str
|
244
|
-
|
245
240
|
CONDA_PACKAGE_S3ROOT: None
|
246
241
|
|
247
242
|
CONDA_PACKAGE_AZUREROOT: None
|
@@ -1,21 +1,22 @@
|
|
1
|
-
|
2
|
-
#
|
3
|
-
# MF version: 2.12.
|
4
|
-
# Generated on 2024-11-
|
5
|
-
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.899445 #
|
5
|
+
######################################################################################################
|
6
6
|
|
7
7
|
from __future__ import annotations
|
8
8
|
|
9
9
|
import typing
|
10
10
|
if typing.TYPE_CHECKING:
|
11
|
-
import typing
|
12
|
-
import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator
|
13
|
-
import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core
|
14
11
|
import metaflow.plugins.cards.component_serializer
|
15
|
-
import
|
12
|
+
import metaflow
|
13
|
+
import typing
|
14
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core
|
16
15
|
import metaflow.events
|
17
16
|
import metaflow.metaflow_current
|
18
|
-
import metaflow
|
17
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator
|
18
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
|
19
|
+
|
19
20
|
|
20
21
|
TYPE_CHECKING: bool
|
21
22
|
|
@@ -224,10 +225,10 @@ class Current(object, metaclass=type):
|
|
224
225
|
"""
|
225
226
|
...
|
226
227
|
@property
|
227
|
-
def graph(
|
228
|
+
def graph(self):
|
228
229
|
...
|
229
230
|
@property
|
230
|
-
def model(self) -> "
|
231
|
+
def model(self) -> "metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core.ModelSerializer":
|
231
232
|
"""
|
232
233
|
(only in the presence of the @model decorator)
|
233
234
|
|
@@ -306,36 +307,7 @@ class Current(object, metaclass=type):
|
|
306
307
|
"""
|
307
308
|
...
|
308
309
|
@property
|
309
|
-
def
|
310
|
-
"""
|
311
|
-
(only in the presence of the @parallel decorator)
|
312
|
-
|
313
|
-
Returns a namedtuple with relevant information about the parallel task.
|
314
|
-
|
315
|
-
Returns
|
316
|
-
-------
|
317
|
-
Parallel
|
318
|
-
`namedtuple` with the following fields:
|
319
|
-
- main_ip (`str`)
|
320
|
-
The IP address of the control task.
|
321
|
-
- num_nodes (`int`)
|
322
|
-
The total number of tasks created by @parallel
|
323
|
-
- node_index (`int`)
|
324
|
-
The index of the current task in all the @parallel tasks.
|
325
|
-
- control_task_id (`Optional[str]`)
|
326
|
-
The task ID of the control task. Available to all tasks.
|
327
|
-
"""
|
328
|
-
...
|
329
|
-
@property
|
330
|
-
def is_parallel(self) -> bool:
|
331
|
-
"""
|
332
|
-
(only in the presence of the @parallel decorator)
|
333
|
-
|
334
|
-
True if the current step is a @parallel step.
|
335
|
-
"""
|
336
|
-
...
|
337
|
-
@property
|
338
|
-
def checkpoint(self) -> "metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator.CurrentCheckpointer":
|
310
|
+
def checkpoint(self) -> "metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator.CurrentCheckpointer":
|
339
311
|
"""
|
340
312
|
(only in the presence of the @checkpoint decorator)
|
341
313
|
|
@@ -396,7 +368,7 @@ class Current(object, metaclass=type):
|
|
396
368
|
"""
|
397
369
|
...
|
398
370
|
@property
|
399
|
-
def huggingface_hub(self) -> "
|
371
|
+
def huggingface_hub(self) -> "metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator.HuggingfaceRegistry":
|
400
372
|
"""
|
401
373
|
(only in the presence of the @huggingface_hub decorator)
|
402
374
|
|
@@ -465,9 +437,38 @@ class Current(object, metaclass=type):
|
|
465
437
|
"""
|
466
438
|
...
|
467
439
|
@property
|
440
|
+
def parallel(self) -> "metaflow.metaflow_current.Parallel":
|
441
|
+
"""
|
442
|
+
(only in the presence of the @parallel decorator)
|
443
|
+
|
444
|
+
Returns a namedtuple with relevant information about the parallel task.
|
445
|
+
|
446
|
+
Returns
|
447
|
+
-------
|
448
|
+
Parallel
|
449
|
+
`namedtuple` with the following fields:
|
450
|
+
- main_ip (`str`)
|
451
|
+
The IP address of the control task.
|
452
|
+
- num_nodes (`int`)
|
453
|
+
The total number of tasks created by @parallel
|
454
|
+
- node_index (`int`)
|
455
|
+
The index of the current task in all the @parallel tasks.
|
456
|
+
- control_task_id (`Optional[str]`)
|
457
|
+
The task ID of the control task. Available to all tasks.
|
458
|
+
"""
|
459
|
+
...
|
460
|
+
@property
|
461
|
+
def is_parallel(self) -> bool:
|
462
|
+
"""
|
463
|
+
(only in the presence of the @parallel decorator)
|
464
|
+
|
465
|
+
True if the current step is a @parallel step.
|
466
|
+
"""
|
467
|
+
...
|
468
|
+
@property
|
468
469
|
def trigger(self) -> "metaflow.events.Trigger":
|
469
470
|
"""
|
470
|
-
(only in the presence of the @
|
471
|
+
(only in the presence of the @trigger, or @trigger_on_finish decorators)
|
471
472
|
|
472
473
|
Returns `Trigger` if the current run is triggered by an event
|
473
474
|
|
@@ -0,0 +1,6 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.804706 #
|
5
|
+
######################################################################################################
|
6
|
+
|
@@ -0,0 +1,6 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.813004 #
|
5
|
+
######################################################################################################
|
6
|
+
|
@@ -0,0 +1,6 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.813069 #
|
5
|
+
######################################################################################################
|
6
|
+
|
@@ -0,0 +1,6 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.813128 #
|
5
|
+
######################################################################################################
|
6
|
+
|
@@ -0,0 +1,6 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.813185 #
|
5
|
+
######################################################################################################
|
6
|
+
|
@@ -0,0 +1,129 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.876821 #
|
5
|
+
######################################################################################################
|
6
|
+
|
7
|
+
from __future__ import annotations
|
8
|
+
|
9
|
+
import metaflow
|
10
|
+
import typing
|
11
|
+
if typing.TYPE_CHECKING:
|
12
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core
|
13
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
|
14
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
|
15
|
+
import metaflow.datastore.datastore_storage
|
16
|
+
|
17
|
+
from ..exceptions import KeyNotCompatibleWithObjectException as KeyNotCompatibleWithObjectException
|
18
|
+
from ..utils.identity_utils import pathspec_hash as pathspec_hash
|
19
|
+
from ..utils.general import replace_start_and_end_slash as replace_start_and_end_slash
|
20
|
+
from ..datastore.core import allow_safe as allow_safe
|
21
|
+
from ..datastore.core import DatastoreInterface as DatastoreInterface
|
22
|
+
from ..datastore.core import ObjectStorage as ObjectStorage
|
23
|
+
from ..datastore.core import STORAGE_FORMATS as STORAGE_FORMATS
|
24
|
+
from ..datastore.exceptions import DatastoreReadInitException as DatastoreReadInitException
|
25
|
+
from ..datastore.exceptions import DatastoreWriteInitException as DatastoreWriteInitException
|
26
|
+
from ..datastore.exceptions import DatastoreNotReadyException as DatastoreNotReadyException
|
27
|
+
from ..datastructures import CheckpointArtifact as CheckpointArtifact
|
28
|
+
from ..datastore.utils import safe_serialize as safe_serialize
|
29
|
+
|
30
|
+
DEFAULT_NAME: str
|
31
|
+
|
32
|
+
CHECKPOINTS_STORAGE_PREFIX: str
|
33
|
+
|
34
|
+
DEFAULT_STORAGE_FORMAT: str
|
35
|
+
|
36
|
+
ARTIFACT_STORE_NAME: str
|
37
|
+
|
38
|
+
METADATA_STORE_NAME: str
|
39
|
+
|
40
|
+
ARTIFACT_METADATA_STORE_NAME: str
|
41
|
+
|
42
|
+
class CheckpointsPathComponents(tuple, metaclass=type):
|
43
|
+
"""
|
44
|
+
CheckpointsPathComponents(flow_name, step_name, scope, task_identifier, pathspec_hash, attempt, name, version_id, is_metadata, key_name, root_prefix)
|
45
|
+
"""
|
46
|
+
@staticmethod
|
47
|
+
def __new__(_cls, flow_name, step_name, scope, task_identifier, pathspec_hash, attempt, name, version_id, is_metadata, key_name, root_prefix):
|
48
|
+
"""
|
49
|
+
Create new instance of CheckpointsPathComponents(flow_name, step_name, scope, task_identifier, pathspec_hash, attempt, name, version_id, is_metadata, key_name, root_prefix)
|
50
|
+
"""
|
51
|
+
...
|
52
|
+
def __repr__(self):
|
53
|
+
"""
|
54
|
+
Return a nicely formatted representation string
|
55
|
+
"""
|
56
|
+
...
|
57
|
+
def __getnewargs__(self):
|
58
|
+
"""
|
59
|
+
Return self as a plain tuple. Used by copy and pickle.
|
60
|
+
"""
|
61
|
+
...
|
62
|
+
...
|
63
|
+
|
64
|
+
def decompose_key_artifact_metadata_store(key) -> CheckpointsPathComponents:
|
65
|
+
...
|
66
|
+
|
67
|
+
def decompose_key_artifact_store(key) -> CheckpointsPathComponents:
|
68
|
+
"""
|
69
|
+
Convert Key into Path Components.
|
70
|
+
PATH COMPONENTS: mf.checkpoints/artifacts/<flow_name>/<step_name>/<scope>/<task_identifier>/<pathspec_hash>.<attempt>.<name>.<version_id>
|
71
|
+
"""
|
72
|
+
...
|
73
|
+
|
74
|
+
def decompose_key_metadata_store(key) -> CheckpointsPathComponents:
|
75
|
+
"""
|
76
|
+
Convert Key into Path Components.
|
77
|
+
PATH COMPONENTS: mf.checkpoints/artifacts/<flow_name>/<step_name>/<scope>/<task_identifier>/<pathspec_hash>.<attempt>.<name>.<version_id>
|
78
|
+
"""
|
79
|
+
...
|
80
|
+
|
81
|
+
class CheckpointDatastore(metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.DatastoreInterface, metaclass=type):
|
82
|
+
"""
|
83
|
+
Consisits of 3 main components:
|
84
|
+
- `artifact_store`: This is where the checkpoint artifacts are stored.
|
85
|
+
- This key to the checkpoint in this store becomes the "key for the checkpoint"
|
86
|
+
- `metadata_store`: This is where the metadata of the checkpoint is stored it based on the currently executing task (path structure resembles that of a metaflow pathspec).
|
87
|
+
- This store helps retrieve information about all the checkpoints stored for a task during the execution.
|
88
|
+
- `artifact_metadatastore`: This is similar to the metadata store but holds a pathstructure similar to the artifact store.
|
89
|
+
- this store helps reverse lookup the Checkpoint metadata object from the checkpoint key.
|
90
|
+
"""
|
91
|
+
@property
|
92
|
+
def metadata_ready(self):
|
93
|
+
...
|
94
|
+
@property
|
95
|
+
def artifact_ready(self):
|
96
|
+
...
|
97
|
+
def set_root_prefix(self, root_prefix):
|
98
|
+
...
|
99
|
+
@classmethod
|
100
|
+
def init_read_store(cls, storage_backend: metaflow.datastore.datastore_storage.DataStoreStorage, pathspec = None, checkpoint_key = None):
|
101
|
+
"""
|
102
|
+
This will initialize the datastore for reading.
|
103
|
+
|
104
|
+
- If there is only the pathspec that's provided then it can mean the user is doing a list operations
|
105
|
+
- if only the checkpoint_key is provided then it can mean the user is trying to load a specific checkpoint
|
106
|
+
"""
|
107
|
+
...
|
108
|
+
def create_key_name(self, *args):
|
109
|
+
...
|
110
|
+
@classmethod
|
111
|
+
def init_write_store(cls, storage_backend: metaflow.datastore.datastore_storage.DataStoreStorage, pathspec, scope, task_identifier):
|
112
|
+
...
|
113
|
+
def save(self, local_path: str, attempt, version_id, name = 'mfchckpt', metadata = {}, set_latest = True, storage_format = 'files') -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact:
|
114
|
+
...
|
115
|
+
def latest(*args, **kwargs):
|
116
|
+
...
|
117
|
+
def load(self, local_path, version_id, attempt, name, storage_format = 'files'):
|
118
|
+
...
|
119
|
+
def load_metadata(self, attempt, version_id, name = 'mfchckpt') -> dict:
|
120
|
+
...
|
121
|
+
def list(self, name = None, attempt = None, within_task = True):
|
122
|
+
...
|
123
|
+
@classmethod
|
124
|
+
def decompose_key(cls, key) -> CheckpointsPathComponents:
|
125
|
+
...
|
126
|
+
def __init__(self, artifact_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, metadata_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, artifact_metadatastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage):
|
127
|
+
...
|
128
|
+
...
|
129
|
+
|
@@ -0,0 +1,26 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.835340 #
|
5
|
+
######################################################################################################
|
6
|
+
|
7
|
+
from __future__ import annotations
|
8
|
+
|
9
|
+
import typing
|
10
|
+
if typing.TYPE_CHECKING:
|
11
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
|
12
|
+
|
13
|
+
from .core import Checkpointer as Checkpointer
|
14
|
+
from .core import WriteResolver as WriteResolver
|
15
|
+
from .core import ReadResolver as ReadResolver
|
16
|
+
from ..datastructures import CheckpointArtifact as CheckpointArtifact
|
17
|
+
|
18
|
+
TYPE_CHECKING: bool
|
19
|
+
|
20
|
+
CHECKPOINT_UID_ENV_VAR_NAME: str
|
21
|
+
|
22
|
+
DEFAULT_NAME: str
|
23
|
+
|
24
|
+
def load_checkpoint(checkpoint: typing.Union[metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact, dict, str], local_path: str):
|
25
|
+
...
|
26
|
+
|
metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi
ADDED
@@ -0,0 +1,156 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.865978 #
|
5
|
+
######################################################################################################
|
6
|
+
|
7
|
+
from __future__ import annotations
|
8
|
+
|
9
|
+
import typing
|
10
|
+
if typing.TYPE_CHECKING:
|
11
|
+
import metaflow
|
12
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
|
13
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
|
14
|
+
import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.core
|
15
|
+
|
16
|
+
from ......metaflow_current import current as current
|
17
|
+
from ......exception import MetaflowException as MetaflowException
|
18
|
+
from .exceptions import CheckpointNotAvailableException as CheckpointNotAvailableException
|
19
|
+
from .exceptions import CheckpointException as CheckpointException
|
20
|
+
from ..utils import flowspec_utils as flowspec_utils
|
21
|
+
from .checkpoint_storage import CheckpointDatastore as CheckpointDatastore
|
22
|
+
from ..datastructures import CheckpointArtifact as CheckpointArtifact
|
23
|
+
from ..datastore.task_utils import init_datastorage_object as init_datastorage_object
|
24
|
+
from ..datastore.task_utils import resolve_storage_backend as resolve_task_storage_backend
|
25
|
+
|
26
|
+
TYPE_CHECKING: bool
|
27
|
+
|
28
|
+
CHECKPOINT_TAG_PREFIX: str
|
29
|
+
|
30
|
+
MAX_HASH_LEN: int
|
31
|
+
|
32
|
+
DEFAULT_NAME: str
|
33
|
+
|
34
|
+
CHECKPOINT_UID_ENV_VAR_NAME: str
|
35
|
+
|
36
|
+
DEFAULT_STORAGE_FORMAT: str
|
37
|
+
|
38
|
+
class Checkpointer(object, metaclass=type):
|
39
|
+
@property
|
40
|
+
def current_version(self):
|
41
|
+
...
|
42
|
+
def set_root_prefix(self, root_prefix):
|
43
|
+
...
|
44
|
+
def __init__(self, datastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore, attempt: int = 0):
|
45
|
+
...
|
46
|
+
def load_metadata(self, version_id: int = None, name = 'mfchckpt') -> dict:
|
47
|
+
...
|
48
|
+
def save(self, path: str, metadata = {}, latest = True, name = 'mfchckpt', storage_format = 'files') -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact:
|
49
|
+
...
|
50
|
+
...
|
51
|
+
|
52
|
+
class CheckpointLoadPolicy(object, metaclass=type):
|
53
|
+
@classmethod
|
54
|
+
def fresh(cls, datastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore, flow: "metaflow.FlowSpec") -> typing.Optional[metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact]:
|
55
|
+
"""
|
56
|
+
```python
|
57
|
+
@checkpoint(load_policy="fresh")
|
58
|
+
```
|
59
|
+
While in `fresh` mode, we want to load the "latest" checkpoint from
|
60
|
+
what ever task is executing at the current memoment.
|
61
|
+
|
62
|
+
The behavior is will be such that 1st attempt of any task will not load
|
63
|
+
any checkpoint and there after it will load the checkpoint from the previous
|
64
|
+
attempt of the task (ala the lastest checkpoint within the task).
|
65
|
+
"""
|
66
|
+
...
|
67
|
+
@classmethod
|
68
|
+
def eager(cls, datastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore, flow: "metaflow.FlowSpec") -> typing.Optional[metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact]:
|
69
|
+
"""
|
70
|
+
```python
|
71
|
+
@checkpoint(load_policy="eager")
|
72
|
+
```
|
73
|
+
While in `eager` mode, we want to load the "latest" checkpoint ever
|
74
|
+
written at the "scope" level based on the kind of task that is executing.
|
75
|
+
|
76
|
+
Setting this mode helps "checkpoints leak across executions" for the same task
|
77
|
+
there by allowing a way to reboot the state when new executions start.
|
78
|
+
"""
|
79
|
+
...
|
80
|
+
...
|
81
|
+
|
82
|
+
class ScopeResolver(object, metaclass=type):
|
83
|
+
@classmethod
|
84
|
+
def from_namespace(cls):
|
85
|
+
...
|
86
|
+
@classmethod
|
87
|
+
def from_tags(cls, tags):
|
88
|
+
...
|
89
|
+
...
|
90
|
+
|
91
|
+
class ReadResolver(object, metaclass=type):
|
92
|
+
"""
|
93
|
+
Responsible for instantiating the `CheckpointDatastore` during read operations
|
94
|
+
based on different context's.
|
95
|
+
"""
|
96
|
+
@classmethod
|
97
|
+
def from_key(cls, checkpoint_key):
|
98
|
+
"""
|
99
|
+
"""
|
100
|
+
...
|
101
|
+
@classmethod
|
102
|
+
def from_key_and_pathspec(cls, pathspec, checkpoint_key):
|
103
|
+
"""
|
104
|
+
"""
|
105
|
+
...
|
106
|
+
@classmethod
|
107
|
+
def from_key_and_run(cls, run, checkpoint_key) -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore:
|
108
|
+
"""
|
109
|
+
"""
|
110
|
+
...
|
111
|
+
...
|
112
|
+
|
113
|
+
class WriteResolver(object, metaclass=type):
|
114
|
+
"""
|
115
|
+
Responsible for instantiating the `CheckpointDatastore` and the subsequent
|
116
|
+
`_checkpointer_uid` which can instantiate the `Checkpointer` object outside
|
117
|
+
of the metaflow context.
|
118
|
+
"""
|
119
|
+
@classmethod
|
120
|
+
def can_resolve_from_envionment(cls):
|
121
|
+
...
|
122
|
+
@classmethod
|
123
|
+
def decompose_checkpoint_id(cls, checkpoint_id):
|
124
|
+
...
|
125
|
+
@classmethod
|
126
|
+
def resolver_info(cls, flow, run, step, taskid, taskidf, scope, attempt):
|
127
|
+
...
|
128
|
+
@classmethod
|
129
|
+
def construct_checkpoint_id(cls, resolver_info: ResolverInfo):
|
130
|
+
...
|
131
|
+
@classmethod
|
132
|
+
def from_environment(cls):
|
133
|
+
...
|
134
|
+
@classmethod
|
135
|
+
def from_run(cls, run: "metaflow.FlowSpec", scope: str, task_identifier: typing.Optional[str] = None, gang_scheduled_task = False):
|
136
|
+
"""
|
137
|
+
The task-identifier gets computed in the Metaflow main process with the
|
138
|
+
i.e. in the decorator. The pathspec we choose to write the metadata store
|
139
|
+
depends on if the task is being gang scheduled or not.
|
140
|
+
"""
|
141
|
+
...
|
142
|
+
...
|
143
|
+
|
144
|
+
class CheckpointReferenceResolver(object, metaclass=type):
|
145
|
+
"""
|
146
|
+
Resolve the Metaflow checkpoint object based on the flow artifact reference
|
147
|
+
or key; Used for lineage derivation.
|
148
|
+
"""
|
149
|
+
@classmethod
|
150
|
+
def from_key(cls, flow, checkpoint_key):
|
151
|
+
"""
|
152
|
+
Used by lineage derivation
|
153
|
+
"""
|
154
|
+
...
|
155
|
+
...
|
156
|
+
|
@@ -0,0 +1,25 @@
|
|
1
|
+
######################################################################################################
|
2
|
+
# Auto-generated Metaflow stub file #
|
3
|
+
# MF version: 2.12.30.2+obcheckpoint(0.1.4);ob(v1) #
|
4
|
+
# Generated on 2024-11-21T22:12:20.835614 #
|
5
|
+
######################################################################################################
|
6
|
+
|
7
|
+
from __future__ import annotations
|
8
|
+
|
9
|
+
import metaflow
|
10
|
+
import typing
|
11
|
+
if typing.TYPE_CHECKING:
|
12
|
+
import metaflow.exception
|
13
|
+
|
14
|
+
from ......exception import MetaflowException as MetaflowException
|
15
|
+
|
16
|
+
class CheckpointNotAvailableException(metaflow.exception.MetaflowException, metaclass=type):
|
17
|
+
def __init__(self, message):
|
18
|
+
...
|
19
|
+
...
|
20
|
+
|
21
|
+
class CheckpointException(metaflow.exception.MetaflowException, metaclass=type):
|
22
|
+
def __init__(self, message):
|
23
|
+
...
|
24
|
+
...
|
25
|
+
|