ob-metaflow-stubs 6.0.3.117__py2.py3-none-any.whl → 6.0.3.118__py2.py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (175) hide show
  1. metaflow-stubs/__init__.pyi +319 -2739
  2. metaflow-stubs/cards.pyi +19 -473
  3. metaflow-stubs/cli.pyi +17 -81
  4. metaflow-stubs/client/__init__.pyi +19 -1113
  5. metaflow-stubs/client/core.pyi +18 -158
  6. metaflow-stubs/client/filecache.pyi +8 -12
  7. metaflow-stubs/clone_util.pyi +6 -26
  8. metaflow-stubs/events.pyi +6 -5
  9. metaflow-stubs/exception.pyi +8 -6
  10. metaflow-stubs/flowspec.pyi +22 -106
  11. metaflow-stubs/generated_for.txt +1 -1
  12. metaflow-stubs/includefile.pyi +16 -564
  13. metaflow-stubs/info_file.pyi +6 -5
  14. metaflow-stubs/metadata_provider/__init__.pyi +16 -0
  15. metaflow-stubs/metadata_provider/heartbeat.pyi +34 -0
  16. metaflow-stubs/{metadata → metadata_provider}/metadata.pyi +10 -22
  17. metaflow-stubs/metadata_provider/util.pyi +19 -0
  18. metaflow-stubs/metaflow_config.pyi +8 -13
  19. metaflow-stubs/metaflow_current.pyi +134 -133
  20. metaflow-stubs/mf_extensions/__init__.pyi +6 -0
  21. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +6 -0
  22. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +6 -0
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +6 -0
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +6 -0
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +129 -0
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +26 -0
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +156 -0
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +25 -0
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +111 -0
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +6 -0
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +188 -0
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +30 -0
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +28 -0
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +19 -0
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +115 -0
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +40 -0
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +6 -0
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +71 -0
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +56 -0
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +67 -0
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +49 -0
  42. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +6 -0
  43. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +6 -0
  44. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +130 -0
  45. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +42 -0
  46. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +6 -0
  47. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +17 -0
  48. metaflow-stubs/mflog/__init__.pyi +6 -0
  49. metaflow-stubs/mflog/mflog.pyi +52 -5
  50. metaflow-stubs/multicore_utils.pyi +6 -5
  51. metaflow-stubs/parameters.pyi +13 -23
  52. metaflow-stubs/plugins/__init__.pyi +52 -165
  53. metaflow-stubs/plugins/airflow/__init__.pyi +9 -5
  54. metaflow-stubs/plugins/airflow/airflow_utils.pyi +7 -6
  55. metaflow-stubs/plugins/airflow/exception.pyi +7 -11
  56. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +10 -97
  57. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +9 -30
  58. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +9 -40
  59. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +9 -40
  60. metaflow-stubs/plugins/argo/__init__.pyi +12 -5
  61. metaflow-stubs/plugins/argo/argo_client.pyi +8 -26
  62. metaflow-stubs/plugins/argo/argo_events.pyi +7 -11
  63. metaflow-stubs/plugins/argo/argo_workflows.pyi +18 -130
  64. metaflow-stubs/plugins/argo/argo_workflows_cli.pyi +22 -460
  65. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +12 -404
  66. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +63 -448
  67. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +165 -0
  68. metaflow-stubs/plugins/aws/__init__.pyi +11 -5
  69. metaflow-stubs/plugins/aws/aws_client.pyi +6 -5
  70. metaflow-stubs/plugins/aws/aws_utils.pyi +6 -11
  71. metaflow-stubs/plugins/aws/batch/__init__.pyi +9 -5
  72. metaflow-stubs/plugins/aws/batch/batch.pyi +10 -55
  73. metaflow-stubs/plugins/aws/batch/batch_client.pyi +7 -11
  74. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +15 -140
  75. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +7 -5
  76. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +10 -21
  77. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +9 -5
  78. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +6 -5
  79. metaflow-stubs/plugins/aws/step_functions/production_token.pyi +6 -5
  80. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +7 -5
  81. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +11 -65
  82. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +6 -5
  83. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +52 -292
  84. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +127 -0
  85. metaflow-stubs/plugins/azure/__init__.pyi +12 -7
  86. metaflow-stubs/plugins/azure/azure_credential.pyi +6 -5
  87. metaflow-stubs/plugins/azure/azure_exceptions.pyi +7 -11
  88. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +11 -24
  89. metaflow-stubs/plugins/azure/azure_utils.pyi +11 -29
  90. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +8 -23
  91. metaflow-stubs/plugins/azure/includefile_support.pyi +7 -17
  92. metaflow-stubs/plugins/cards/__init__.pyi +15 -5
  93. metaflow-stubs/plugins/cards/card_cli.pyi +22 -491
  94. metaflow-stubs/plugins/cards/card_client.pyi +13 -75
  95. metaflow-stubs/plugins/cards/card_creator.pyi +7 -10
  96. metaflow-stubs/plugins/cards/card_datastore.pyi +10 -18
  97. metaflow-stubs/plugins/cards/card_decorator.pyi +10 -126
  98. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +13 -81
  99. metaflow-stubs/plugins/cards/card_modules/basic.pyi +13 -96
  100. metaflow-stubs/plugins/cards/card_modules/card.pyi +6 -5
  101. metaflow-stubs/plugins/cards/card_modules/components.pyi +24 -107
  102. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +6 -5
  103. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +6 -12
  104. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +11 -88
  105. metaflow-stubs/plugins/cards/card_resolver.pyi +6 -49
  106. metaflow-stubs/plugins/cards/component_serializer.pyi +13 -63
  107. metaflow-stubs/plugins/cards/exception.pyi +7 -11
  108. metaflow-stubs/plugins/catch_decorator.pyi +10 -30
  109. metaflow-stubs/plugins/datatools/__init__.pyi +13 -392
  110. metaflow-stubs/plugins/datatools/local.pyi +7 -11
  111. metaflow-stubs/plugins/datatools/s3/__init__.pyi +19 -653
  112. metaflow-stubs/plugins/datatools/s3/s3.pyi +16 -264
  113. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +7 -10
  114. metaflow-stubs/plugins/datatools/s3/s3util.pyi +6 -11
  115. metaflow-stubs/plugins/debug_logger.pyi +7 -5
  116. metaflow-stubs/plugins/debug_monitor.pyi +7 -5
  117. metaflow-stubs/plugins/environment_decorator.pyi +7 -5
  118. metaflow-stubs/plugins/events_decorator.pyi +8 -14
  119. metaflow-stubs/plugins/frameworks/__init__.pyi +7 -5
  120. metaflow-stubs/plugins/frameworks/pytorch.pyi +8 -45
  121. metaflow-stubs/plugins/gcp/__init__.pyi +11 -7
  122. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +11 -24
  123. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +7 -11
  124. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +6 -5
  125. metaflow-stubs/plugins/gcp/gs_utils.pyi +8 -20
  126. metaflow-stubs/plugins/gcp/includefile_support.pyi +7 -17
  127. metaflow-stubs/plugins/kubernetes/__init__.pyi +12 -5
  128. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +6 -10
  129. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +10 -38
  130. metaflow-stubs/plugins/kubernetes/kubernetes_cli.pyi +16 -155
  131. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +7 -11
  132. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +19 -142
  133. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +7 -11
  134. metaflow-stubs/plugins/logs_cli.pyi +10 -9
  135. metaflow-stubs/plugins/package_cli.pyi +7 -5
  136. metaflow-stubs/plugins/parallel_decorator.pyi +11 -59
  137. metaflow-stubs/plugins/perimeters.pyi +6 -11
  138. metaflow-stubs/plugins/project_decorator.pyi +8 -14
  139. metaflow-stubs/plugins/pypi/__init__.pyi +12 -11
  140. metaflow-stubs/plugins/pypi/conda_decorator.pyi +8 -27
  141. metaflow-stubs/plugins/pypi/conda_environment.pyi +14 -20
  142. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +7 -5
  143. metaflow-stubs/plugins/pypi/pypi_environment.pyi +7 -39
  144. metaflow-stubs/plugins/pypi/utils.pyi +7 -11
  145. metaflow-stubs/plugins/resources_decorator.pyi +7 -5
  146. metaflow-stubs/plugins/retry_decorator.pyi +7 -11
  147. metaflow-stubs/plugins/secrets/__init__.pyi +9 -5
  148. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +8 -13
  149. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +7 -11
  150. metaflow-stubs/plugins/storage_executor.pyi +6 -11
  151. metaflow-stubs/plugins/tag_cli.pyi +14 -396
  152. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +8 -33
  153. metaflow-stubs/plugins/timeout_decorator.pyi +8 -12
  154. metaflow-stubs/procpoll.pyi +7 -5
  155. metaflow-stubs/profilers/__init__.pyi +7 -11
  156. metaflow-stubs/pylint_wrapper.pyi +7 -11
  157. metaflow-stubs/runner/__init__.pyi +13 -5
  158. metaflow-stubs/runner/deployer.pyi +99 -223
  159. metaflow-stubs/runner/deployer_impl.pyi +87 -0
  160. metaflow-stubs/runner/metaflow_runner.pyi +24 -508
  161. metaflow-stubs/runner/nbdeploy.pyi +16 -60
  162. metaflow-stubs/runner/nbrun.pyi +11 -148
  163. metaflow-stubs/runner/subprocess_manager.pyi +9 -10
  164. metaflow-stubs/runner/utils.pyi +44 -9
  165. metaflow-stubs/system/__init__.pyi +9 -107
  166. metaflow-stubs/system/system_logger.pyi +6 -15
  167. metaflow-stubs/system/system_monitor.pyi +6 -16
  168. metaflow-stubs/tagging_util.pyi +6 -10
  169. metaflow-stubs/tuple_util.pyi +6 -5
  170. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/METADATA +1 -1
  171. ob_metaflow_stubs-6.0.3.118.dist-info/RECORD +174 -0
  172. metaflow-stubs/metadata/util.pyi +0 -18
  173. ob_metaflow_stubs-6.0.3.117.dist-info/RECORD +0 -140
  174. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/WHEEL +0 -0
  175. {ob_metaflow_stubs-6.0.3.117.dist-info → ob_metaflow_stubs-6.0.3.118.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,156 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.900747 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import typing
10
+ if typing.TYPE_CHECKING:
11
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
12
+ import metaflow
13
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
14
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.core
15
+
16
+ from ......metaflow_current import current as current
17
+ from ......exception import MetaflowException as MetaflowException
18
+ from .exceptions import CheckpointNotAvailableException as CheckpointNotAvailableException
19
+ from .exceptions import CheckpointException as CheckpointException
20
+ from ..utils import flowspec_utils as flowspec_utils
21
+ from .checkpoint_storage import CheckpointDatastore as CheckpointDatastore
22
+ from ..datastructures import CheckpointArtifact as CheckpointArtifact
23
+ from ..datastore.task_utils import init_datastorage_object as init_datastorage_object
24
+ from ..datastore.task_utils import resolve_storage_backend as resolve_task_storage_backend
25
+
26
+ TYPE_CHECKING: bool
27
+
28
+ CHECKPOINT_TAG_PREFIX: str
29
+
30
+ MAX_HASH_LEN: int
31
+
32
+ DEFAULT_NAME: str
33
+
34
+ CHECKPOINT_UID_ENV_VAR_NAME: str
35
+
36
+ DEFAULT_STORAGE_FORMAT: str
37
+
38
+ class Checkpointer(object, metaclass=type):
39
+ @property
40
+ def current_version(self):
41
+ ...
42
+ def set_root_prefix(self, root_prefix):
43
+ ...
44
+ def __init__(self, datastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore, attempt: int = 0):
45
+ ...
46
+ def load_metadata(self, version_id: int = None, name = 'mfchckpt') -> dict:
47
+ ...
48
+ def save(self, path: str, metadata = {}, latest = True, name = 'mfchckpt', storage_format = 'files') -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact:
49
+ ...
50
+ ...
51
+
52
+ class CheckpointLoadPolicy(object, metaclass=type):
53
+ @classmethod
54
+ def fresh(cls, datastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore, flow: "metaflow.FlowSpec") -> typing.Optional[metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact]:
55
+ """
56
+ ```python
57
+ @checkpoint(load_policy="fresh")
58
+ ```
59
+ While in `fresh` mode, we want to load the "latest" checkpoint from
60
+ what ever task is executing at the current memoment.
61
+
62
+ The behavior is will be such that 1st attempt of any task will not load
63
+ any checkpoint and there after it will load the checkpoint from the previous
64
+ attempt of the task (ala the lastest checkpoint within the task).
65
+ """
66
+ ...
67
+ @classmethod
68
+ def eager(cls, datastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore, flow: "metaflow.FlowSpec") -> typing.Optional[metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact]:
69
+ """
70
+ ```python
71
+ @checkpoint(load_policy="eager")
72
+ ```
73
+ While in `eager` mode, we want to load the "latest" checkpoint ever
74
+ written at the "scope" level based on the kind of task that is executing.
75
+
76
+ Setting this mode helps "checkpoints leak across executions" for the same task
77
+ there by allowing a way to reboot the state when new executions start.
78
+ """
79
+ ...
80
+ ...
81
+
82
+ class ScopeResolver(object, metaclass=type):
83
+ @classmethod
84
+ def from_namespace(cls):
85
+ ...
86
+ @classmethod
87
+ def from_tags(cls, tags):
88
+ ...
89
+ ...
90
+
91
+ class ReadResolver(object, metaclass=type):
92
+ """
93
+ Responsible for instantiating the `CheckpointDatastore` during read operations
94
+ based on different context's.
95
+ """
96
+ @classmethod
97
+ def from_key(cls, checkpoint_key):
98
+ """
99
+ """
100
+ ...
101
+ @classmethod
102
+ def from_key_and_pathspec(cls, pathspec, checkpoint_key):
103
+ """
104
+ """
105
+ ...
106
+ @classmethod
107
+ def from_key_and_run(cls, run, checkpoint_key) -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage.CheckpointDatastore:
108
+ """
109
+ """
110
+ ...
111
+ ...
112
+
113
+ class WriteResolver(object, metaclass=type):
114
+ """
115
+ Responsible for instantiating the `CheckpointDatastore` and the subsequent
116
+ `_checkpointer_uid` which can instantiate the `Checkpointer` object outside
117
+ of the metaflow context.
118
+ """
119
+ @classmethod
120
+ def can_resolve_from_envionment(cls):
121
+ ...
122
+ @classmethod
123
+ def decompose_checkpoint_id(cls, checkpoint_id):
124
+ ...
125
+ @classmethod
126
+ def resolver_info(cls, flow, run, step, taskid, taskidf, scope, attempt):
127
+ ...
128
+ @classmethod
129
+ def construct_checkpoint_id(cls, resolver_info: ResolverInfo):
130
+ ...
131
+ @classmethod
132
+ def from_environment(cls):
133
+ ...
134
+ @classmethod
135
+ def from_run(cls, run: "metaflow.FlowSpec", scope: str, task_identifier: typing.Optional[str] = None, gang_scheduled_task = False):
136
+ """
137
+ The task-identifier gets computed in the Metaflow main process with the
138
+ i.e. in the decorator. The pathspec we choose to write the metadata store
139
+ depends on if the task is being gang scheduled or not.
140
+ """
141
+ ...
142
+ ...
143
+
144
+ class CheckpointReferenceResolver(object, metaclass=type):
145
+ """
146
+ Resolve the Metaflow checkpoint object based on the flow artifact reference
147
+ or key; Used for lineage derivation.
148
+ """
149
+ @classmethod
150
+ def from_key(cls, flow, checkpoint_key):
151
+ """
152
+ Used by lineage derivation
153
+ """
154
+ ...
155
+ ...
156
+
@@ -0,0 +1,25 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.869828 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.exception
13
+
14
+ from ......exception import MetaflowException as MetaflowException
15
+
16
+ class CheckpointNotAvailableException(metaflow.exception.MetaflowException, metaclass=type):
17
+ def __init__(self, message):
18
+ ...
19
+ ...
20
+
21
+ class CheckpointException(metaflow.exception.MetaflowException, metaclass=type):
22
+ def __init__(self, message):
23
+ ...
24
+ ...
25
+
@@ -0,0 +1,111 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.846265 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import typing
10
+ if typing.TYPE_CHECKING:
11
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
12
+ import metaflow
13
+
14
+ from ..datastructures import CheckpointArtifact as CheckpointArtifact
15
+ from .constructors import load_checkpoint as load_checkpoint
16
+ from .exceptions import CheckpointNotAvailableException as CheckpointNotAvailableException
17
+
18
+ TYPE_CHECKING: bool
19
+
20
+ CHECKPOINT_UID_ENV_VAR_NAME: str
21
+
22
+ DEFAULT_NAME: str
23
+
24
+ TASK_CHECKPOINTS_ARTIFACT_NAME: str
25
+
26
+ DEFAULT_STORAGE_FORMAT: str
27
+
28
+ class Checkpoint(object, metaclass=type):
29
+ def __init__(self, temp_dir_root = None, init_dir = False):
30
+ ...
31
+ @property
32
+ def directory(self):
33
+ ...
34
+ def save(self, path = None, metadata = None, latest = True, name = 'mfchckpt', storage_format = 'files'):
35
+ """
36
+ Saves the checkpoint to the datastore
37
+
38
+ Parameters
39
+ ----------
40
+ path : Optional[Union[str, os.PathLike]], default: None
41
+ The path to save the checkpoint. Accepts a file path or a directory path.
42
+ - If a directory path is provided, all the contents within that directory will be saved.
43
+ When a checkpoint is reloaded during task retries, `the current.checkpoint.directory` will
44
+ contain the contents of this directory.
45
+ - If a file path is provided, the file will be directly saved to the datastore (with the same filename).
46
+ When the checkpoint is reloaded during task retries, the file with the same name will be available in the
47
+ `current.checkpoint.directory`.
48
+ - If no path is provided then the `Checkpoint.directory` will be saved as the checkpoint.
49
+
50
+ name : Optional[str], default: "mfchckpt"
51
+ The name of the checkpoint.
52
+
53
+ metadata : Optional[Dict], default: {}
54
+ Any metadata that needs to be saved with the checkpoint.
55
+
56
+ latest : bool, default: True
57
+ If True, the checkpoint will be marked as the latest checkpoint.
58
+ This helps determine if the checkpoint gets loaded when the task restarts.
59
+
60
+ storage_format : str, default: files
61
+ If `tar`, the contents of the directory will be tarred before saving to the datastore.
62
+ If `files`, saves directory directly to the datastore.
63
+ """
64
+ ...
65
+ def __enter__(self):
66
+ ...
67
+ def __exit__(self, exc_type, exc_val, exc_tb):
68
+ ...
69
+ def list(self, name: typing.Optional[str] = None, task: typing.Union["metaflow.Task", str, None] = None, attempt: typing.Union[int, str, None] = None, as_dict: bool = True, within_task: bool = True) -> typing.Iterable[typing.Union[typing.Dict, metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact]]:
70
+ """
71
+ lists the checkpoints in the datastore based on the Task.
72
+ It will always be task scoped.
73
+
74
+ Usage:
75
+ ------
76
+
77
+ ```python
78
+
79
+ Checkpoint().list(name="best") # lists checkpoints in the current task with the name "best"
80
+ Checkpoint().list(task="anotherflow/somerunid/somestep/sometask", name="best") # Identical as the above one but
81
+ Checkpoint().list() # lists all the checkpoints in the current task
82
+
83
+ ```
84
+
85
+ Parameters
86
+ ----------
87
+
88
+ - `name`:
89
+ - name of the checkpoint to filter for
90
+ - `task`:
91
+ - Task object outside the one that is currently set in the `Checkpoint` object; Can be a pathspec string.
92
+ - `attempt`:
93
+ - attempt number of the task (optional filter. If none, then lists all checkpoints from all attempts)
94
+ """
95
+ ...
96
+ def load(self, reference: typing.Union[str, typing.Dict, metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact], path: typing.Optional[str] = None):
97
+ """
98
+ loads a checkpoint reference from the datastore. (resembles a read op)
99
+
100
+ Parameters
101
+ ----------
102
+
103
+ `reference` :
104
+ - can be a string, dict or a CheckpointArtifact object:
105
+ - string: a string reference to the checkpoint (checkpoint key)
106
+ - dict: a dictionary reference to the checkpoint
107
+ - CheckpointArtifact: a CheckpointArtifact object reference to the checkpoint
108
+ """
109
+ ...
110
+ ...
111
+
@@ -0,0 +1,6 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.870461 #
5
+ ######################################################################################################
6
+
@@ -0,0 +1,188 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.901955 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import typing
10
+ if typing.TYPE_CHECKING:
11
+ import metaflow.datastore.datastore_storage
12
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core
13
+
14
+ from ......exception import MetaflowException as MetaflowException
15
+ from ..utils.tar_utils import create_tarball_on_disk as create_tarball_on_disk
16
+ from ..utils.tar_utils import extract_tarball as extract_tarball
17
+ from ..utils.general import safe_serialize as safe_serialize
18
+ from ..exceptions import KeyNotFoundError as KeyNotFoundError
19
+
20
+ STORAGE_INJECTIONS_SINGLE_FILE_SAVE: dict
21
+
22
+ STORAGE_INJECTIONS_MULTIPLE_FILE_SAVE: dict
23
+
24
+ STORAGE_INJECTIONS_LOAD_FILES: dict
25
+
26
+ class DatastoreBlob(tuple, metaclass=type):
27
+ """
28
+ DatastoreBlob(blob, url, path)
29
+ """
30
+ @staticmethod
31
+ def __new__(_cls, blob, url, path):
32
+ """
33
+ Create new instance of DatastoreBlob(blob, url, path)
34
+ """
35
+ ...
36
+ def __repr__(self):
37
+ """
38
+ Return a nicely formatted representation string
39
+ """
40
+ ...
41
+ def __getnewargs__(self):
42
+ """
43
+ Return self as a plain tuple. Used by copy and pickle.
44
+ """
45
+ ...
46
+ ...
47
+
48
+ class ListPathResult(tuple, metaclass=type):
49
+ """
50
+ ListPathResult(full_url, key)
51
+ """
52
+ @staticmethod
53
+ def __new__(_cls, full_url, key):
54
+ """
55
+ Create new instance of ListPathResult(full_url, key)
56
+ """
57
+ ...
58
+ def __repr__(self):
59
+ """
60
+ Return a nicely formatted representation string
61
+ """
62
+ ...
63
+ def __getnewargs__(self):
64
+ """
65
+ Return self as a plain tuple. Used by copy and pickle.
66
+ """
67
+ ...
68
+ ...
69
+
70
+ COMPRESSION_METHOD: None
71
+
72
+ class STORAGE_FORMATS(object, metaclass=type):
73
+ ...
74
+
75
+ def warning_message(message, logger = None, ts = False, prefix = '[@checkpoint][artifact-store]'):
76
+ ...
77
+
78
+ def allow_safe(func):
79
+ ...
80
+
81
+ class ObjectStorage(object, metaclass=type):
82
+ """
83
+ `ObjectStorage` wraps around the DataStoreStorage object and provides the lower level
84
+ storage APIs needed by the subsequent classes. This datastore's main function that
85
+ distingushes it from the DataStoreStorage object is that it manages a generalizable
86
+ path structure over different storage backends (s3, azure, gs, local etc.).
87
+
88
+ This object will be used to create multiple "Logical" datastores for constructs like
89
+ `Checkpoints`, `Models` etc.
90
+
91
+ Usage
92
+ -----
93
+ ```
94
+ storage = ObjectStorage(
95
+ storage_backend,
96
+ root_prefix = "mf.checkpoints",
97
+ path_components = ["artifacts", "MyFlow", "step_a", "cd2312rd12d", "x5avasdhtsdfqw"]
98
+ )
99
+ ```
100
+ """
101
+ def __init__(self, storage_backend: metaflow.datastore.datastore_storage.DataStoreStorage, root_prefix: str, path_components: typing.List):
102
+ ...
103
+ @property
104
+ def path_components(self):
105
+ ...
106
+ def set_full_prefix(self, root_prefix):
107
+ ...
108
+ @staticmethod
109
+ def resolve_root(storage_type):
110
+ ...
111
+ def full_base_url(self, prefix = None):
112
+ ...
113
+ def create_key_name(self, *args):
114
+ ...
115
+ @property
116
+ def datastore_root(self):
117
+ ...
118
+ def resolve_key_relative_path(self, key):
119
+ ...
120
+ def resolve_key_full_url(self, key):
121
+ ...
122
+ def resolve_key_path(self, key):
123
+ ...
124
+ def put(self, key: str, obj: typing.Union[str, bytes], overwrite: bool = False) -> str:
125
+ """
126
+ TODO : [THIS IS TERRIBLY INEFFICIENT]
127
+ """
128
+ ...
129
+ def put_file(self, key: str, path: str, overwrite = False):
130
+ ...
131
+ def put_files(self, key_paths: typing.List[typing.Tuple[str, str]], overwrite = False):
132
+ ...
133
+ def get(self, key) -> DatastoreBlob:
134
+ """
135
+ TODO : [THIS IS TERRIBLY INEFFICIENT]
136
+ """
137
+ ...
138
+ def get_file(self, key):
139
+ ...
140
+ def list_paths(self, keys) -> typing.Iterator[ListPathResult]:
141
+ """
142
+ List all objects in the datastore's `keys` index.
143
+ """
144
+ ...
145
+ ...
146
+
147
+ class DatastoreInterface(object, metaclass=type):
148
+ """
149
+ This is the root abstraction used by any underlying datastores like Checkpoint/Model etc.
150
+ to create the saving/loading mechanics using multiple ObjectStores.
151
+
152
+ The inherited classes require the following implemented:
153
+ - `ROOT_PREFIX` : The root prefix for the datastore such as `mf.checkpoints` or `mf.models`.
154
+ - `init_read_store` : The method to initialize the read store; The inheriting class can compose together any number of `BaseDatastore` objects
155
+ - `init_write_store` : The method to initialize the write store; The inheriting class can compose together any number of `BaseDatastore` objects
156
+ - `save` : The method to save the artifact data.
157
+ - `load` : The method to load the artifact data.
158
+ - `save_metadata` : The method to save the metadata about the artifact.
159
+ - `load_metadata` : The method to load the metadata about the artifact.
160
+ - `list` : The method to list all the artifacts.
161
+ """
162
+ def set_root_prefix(self, root_prefix):
163
+ """
164
+ This function helps ensuring that the root prefix of the datastore
165
+ and it's underlying ObjectStores can change trivially.
166
+ """
167
+ ...
168
+ def save(self, *args, **kwargs):
169
+ ...
170
+ def load(self, *args, **kwargs):
171
+ ...
172
+ def save_metadata(self, *args, **kwargs):
173
+ ...
174
+ def load_metadata(self, *args, **kwargs):
175
+ ...
176
+ def list(self, *args, **kwargs):
177
+ ...
178
+ @classmethod
179
+ def init_read_store(cls, storage_backend: metaflow.datastore.datastore_storage.DataStoreStorage, *args, **kwargs):
180
+ ...
181
+ @classmethod
182
+ def init_write_store(cls, storage_backend: metaflow.datastore.datastore_storage.DataStoreStorage, *args, **kwargs):
183
+ ...
184
+ ...
185
+
186
+ def resolve_root(datastore_type):
187
+ ...
188
+
@@ -0,0 +1,30 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.922357 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.exception
13
+
14
+ from ......exception import MetaflowException as MetaflowException
15
+
16
+ class DatastoreReadInitException(metaflow.exception.MetaflowException, metaclass=type):
17
+ def __init__(self, message):
18
+ ...
19
+ ...
20
+
21
+ class DatastoreWriteInitException(metaflow.exception.MetaflowException, metaclass=type):
22
+ def __init__(self, message):
23
+ ...
24
+ ...
25
+
26
+ class DatastoreNotReadyException(metaflow.exception.MetaflowException, metaclass=type):
27
+ def __init__(self, message):
28
+ ...
29
+ ...
30
+
@@ -0,0 +1,28 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.870511 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.exception
13
+ import metaflow
14
+
15
+ from ......exception import MetaflowException as MetaflowException
16
+ from .core import resolve_root as resolve_root
17
+
18
+ TYPE_CHECKING: bool
19
+
20
+ class UnresolvableDatastoreException(metaflow.exception.MetaflowException, metaclass=type):
21
+ ...
22
+
23
+ def init_datastorage_object():
24
+ ...
25
+
26
+ def resolve_storage_backend(pathspec: typing.Union[str, "metaflow.Task"] = None):
27
+ ...
28
+
@@ -0,0 +1,19 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.922553 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+
10
+
11
+ def is_json_serializable(value):
12
+ """
13
+ Check if value is JSON serializable.
14
+ """
15
+ ...
16
+
17
+ def safe_serialize(data):
18
+ ...
19
+
@@ -0,0 +1,115 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1) #
4
+ # Generated on 2024-11-13T23:34:52.847269 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
+
14
+ from .exceptions import KeyNotCompatibleWithObjectException as KeyNotCompatibleWithObjectException
15
+ from .exceptions import KeyNotCompatibleException as KeyNotCompatibleException
16
+ from .exceptions import IncompatibleObjectTypeException as IncompatibleObjectTypeException
17
+ from .datastore.task_utils import init_datastorage_object as init_datastorage_object
18
+
19
+ class MetaflowDataArtifactReference(object, metaclass=type):
20
+ @property
21
+ def size(self):
22
+ ...
23
+ @property
24
+ def url(self):
25
+ ...
26
+ @property
27
+ def key(self):
28
+ ...
29
+ @property
30
+ def pathspec(self):
31
+ ...
32
+ @property
33
+ def attempt(self):
34
+ ...
35
+ @property
36
+ def created_on(self):
37
+ ...
38
+ @property
39
+ def metadata(self):
40
+ ...
41
+ def __init__(self, **kwargs):
42
+ ...
43
+ def validate(self, data):
44
+ ...
45
+ @classmethod
46
+ def from_dict(cls, data) -> typing.Union["ModelArtifact", "CheckpointArtifact"]:
47
+ ...
48
+ @classmethod
49
+ def hydrate(cls, data: typing.Union["ModelArtifact", "CheckpointArtifact", dict]):
50
+ ...
51
+ def to_dict(self):
52
+ ...
53
+ ...
54
+
55
+ class ModelArtifact(MetaflowDataArtifactReference, metaclass=type):
56
+ def __init__(self, **kwargs):
57
+ ...
58
+ @property
59
+ def blob(self):
60
+ ...
61
+ @property
62
+ def uuid(self):
63
+ ...
64
+ @property
65
+ def serializer(self):
66
+ ...
67
+ @property
68
+ def source(self):
69
+ ...
70
+ @property
71
+ def storage_format(self):
72
+ ...
73
+ @classmethod
74
+ def create(cls, pathspec = None, attempt = None, key = None, url = None, model_uuid = None, metadata = None, storage_format = None, source = None, serializer = None, label = None):
75
+ ...
76
+ ...
77
+
78
+ class CheckpointArtifact(MetaflowDataArtifactReference, metaclass=type):
79
+ @property
80
+ def storage_format(self):
81
+ ...
82
+ @property
83
+ def version_id(self):
84
+ ...
85
+ @property
86
+ def name(self):
87
+ ...
88
+ def __init__(self, **kwargs):
89
+ ...
90
+ ...
91
+
92
+ class Factory(object, metaclass=type):
93
+ @classmethod
94
+ def hydrate(cls, data):
95
+ ...
96
+ @classmethod
97
+ def from_dict(cls, data):
98
+ ...
99
+ @classmethod
100
+ def load(cls, data, local_path, storage_backend):
101
+ ...
102
+ @classmethod
103
+ def object_type_from_key(cls, reference_key):
104
+ ...
105
+ @classmethod
106
+ def load_from_key(cls, key_object, local_path, storage_backend):
107
+ ...
108
+ @classmethod
109
+ def load_metadata_from_key(cls, key_object, storage_backend) -> typing.Union[CheckpointArtifact, ModelArtifact]:
110
+ ...
111
+ ...
112
+
113
+ def load_model(reference: typing.Union[str, MetaflowDataArtifactReference, dict], path: str):
114
+ ...
115
+