ob-metaflow-stubs 6.0.3.156__py2.py3-none-any.whl → 6.0.3.157__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. metaflow-stubs/__init__.pyi +721 -606
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +5 -5
  8. metaflow-stubs/client/filecache.pyi +2 -2
  9. metaflow-stubs/events.pyi +3 -3
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +5 -5
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +4 -4
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +87 -87
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +3 -3
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +4 -4
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +6 -6
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +7 -5
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +11 -5
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +6 -5
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +5 -3
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +95 -0
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +7 -9
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +135 -0
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +33 -5
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +4 -4
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +3 -3
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  63. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  76. metaflow-stubs/multicore_utils.pyi +2 -2
  77. metaflow-stubs/parameters.pyi +4 -4
  78. metaflow-stubs/plugins/__init__.pyi +11 -11
  79. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  80. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  81. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  82. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  83. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  84. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  85. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  86. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  88. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  89. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  90. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  91. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +3 -3
  92. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +4 -4
  93. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  94. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  95. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  96. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  97. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  98. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  99. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +4 -4
  102. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  103. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  104. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  105. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  106. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  107. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +4 -4
  108. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +4 -4
  109. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  110. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  111. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  112. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +4 -4
  113. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  114. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  115. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  116. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  117. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  118. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  119. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  120. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  121. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  123. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  124. metaflow-stubs/plugins/cards/card_modules/components.pyi +4 -4
  125. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  126. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  127. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  128. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  129. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  130. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  131. metaflow-stubs/plugins/catch_decorator.pyi +2 -2
  132. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  133. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  134. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  135. metaflow-stubs/plugins/datatools/s3/s3.pyi +4 -4
  136. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  137. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  138. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  139. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  140. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  141. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  142. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  143. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  144. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +4 -4
  146. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  147. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  148. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  149. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  150. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  152. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  153. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  154. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  155. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  156. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  157. metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
  158. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  159. metaflow-stubs/plugins/perimeters.pyi +2 -2
  160. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  161. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  162. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  163. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  164. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  165. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  166. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  167. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  168. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  169. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  170. metaflow-stubs/plugins/secrets/__init__.pyi +2 -2
  171. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  172. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  173. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  174. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  175. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  176. metaflow-stubs/plugins/timeout_decorator.pyi +2 -2
  177. metaflow-stubs/profilers/__init__.pyi +2 -2
  178. metaflow-stubs/pylint_wrapper.pyi +2 -2
  179. metaflow-stubs/runner/__init__.pyi +2 -2
  180. metaflow-stubs/runner/deployer.pyi +30 -30
  181. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  182. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  183. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  184. metaflow-stubs/runner/nbrun.pyi +2 -2
  185. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  186. metaflow-stubs/runner/utils.pyi +3 -3
  187. metaflow-stubs/system/__init__.pyi +2 -2
  188. metaflow-stubs/system/system_logger.pyi +3 -3
  189. metaflow-stubs/system/system_monitor.pyi +2 -2
  190. metaflow-stubs/tagging_util.pyi +2 -2
  191. metaflow-stubs/tuple_util.pyi +2 -2
  192. metaflow-stubs/user_configs/__init__.pyi +2 -2
  193. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  194. metaflow-stubs/user_configs/config_options.pyi +4 -4
  195. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  196. {ob_metaflow_stubs-6.0.3.156.dist-info → ob_metaflow_stubs-6.0.3.157.dist-info}/METADATA +1 -1
  197. ob_metaflow_stubs-6.0.3.157.dist-info/RECORD +200 -0
  198. ob_metaflow_stubs-6.0.3.156.dist-info/RECORD +0 -198
  199. {ob_metaflow_stubs-6.0.3.156.dist-info → ob_metaflow_stubs-6.0.3.157.dist-info}/WHEEL +0 -0
  200. {ob_metaflow_stubs-6.0.3.156.dist-info → ob_metaflow_stubs-6.0.3.157.dist-info}/top_level.txt +0 -0
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.109890 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.493382 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,18 +1,18 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.147245 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.535062 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import metaflow
10
9
  import typing
10
+ import metaflow
11
11
  if typing.TYPE_CHECKING:
12
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core
13
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
14
12
  import metaflow.datastore.datastore_storage
13
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
15
14
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
15
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core
16
16
 
17
17
  from ..exceptions import KeyNotCompatibleWithObjectException as KeyNotCompatibleWithObjectException
18
18
  from ..utils.identity_utils import pathspec_hash as pathspec_hash
@@ -123,6 +123,8 @@ class CheckpointDatastore(metaflow.mf_extensions.obcheckpoint.plugins.machine_le
123
123
  @classmethod
124
124
  def decompose_key(cls, key) -> CheckpointsPathComponents:
125
125
  ...
126
+ def __str__(self):
127
+ ...
126
128
  def __init__(self, artifact_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, metadata_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, artifact_metadatastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage):
127
129
  ...
128
130
  ...
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.079263 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.461128 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,17 +1,17 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.105908 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.490728 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow
12
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.core
13
11
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
14
12
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.core
14
+ import metaflow
15
15
 
16
16
  from ......metaflow_current import current as current
17
17
  from ......exception import MetaflowException as MetaflowException
@@ -22,6 +22,7 @@ from .checkpoint_storage import CheckpointDatastore as CheckpointDatastore
22
22
  from ..datastructures import CheckpointArtifact as CheckpointArtifact
23
23
  from ..datastore.task_utils import init_datastorage_object as init_datastorage_object
24
24
  from ..datastore.task_utils import resolve_storage_backend as resolve_task_storage_backend
25
+ from ..datastore.task_utils import storage_backend_from_flow as storage_backend_from_flow
25
26
 
26
27
  TYPE_CHECKING: bool
27
28
 
@@ -45,6 +46,8 @@ class Checkpointer(object, metaclass=type):
45
46
  ...
46
47
  def load_metadata(self, version_id: int = None, name = 'mfchckpt') -> dict:
47
48
  ...
49
+ def artifact_id(self, name: str, version_id: int = None):
50
+ ...
48
51
  def save(self, path: str, metadata = {}, latest = True, name = 'mfchckpt', storage_format = 'files') -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact:
49
52
  ...
50
53
  ...
@@ -88,6 +91,9 @@ class ScopeResolver(object, metaclass=type):
88
91
  ...
89
92
  ...
90
93
 
94
+ def warning_message(message, logger = None, ts = False, prefix = '[@checkpoint][warning]'):
95
+ ...
96
+
91
97
  class ReadResolver(object, metaclass=type):
92
98
  """
93
99
  Responsible for instantiating the `CheckpointDatastore` during read operations
@@ -102,7 +108,7 @@ class ReadResolver(object, metaclass=type):
102
108
  """
103
109
  ...
104
110
  @classmethod
105
- def from_key_and_pathspec(cls, pathspec, checkpoint_key):
111
+ def from_checkpoint(cls, checkpoint: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact):
106
112
  """
107
113
  """
108
114
  ...
@@ -1,18 +1,18 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.081364 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.463289 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import metaflow
10
9
  import typing
10
+ import metaflow
11
11
  if typing.TYPE_CHECKING:
12
- import metaflow
13
- import os
14
12
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
+ import os
15
14
  import metaflow.decorators
15
+ import metaflow
16
16
 
17
17
  from .exceptions import CheckpointException as CheckpointException
18
18
  from ..utils import flowspec_utils as flowspec_utils
@@ -28,6 +28,7 @@ from .constructors import load_checkpoint as load_checkpoint
28
28
  from .core import ScopeResolver as ScopeResolver
29
29
  from .core import CheckpointLoadPolicy as CheckpointLoadPolicy
30
30
  from ..datastructures import CheckpointArtifact as CheckpointArtifact
31
+ from ..datastore.decorator import set_datastore_context as set_datastore_context
31
32
  from .final_api import Checkpoint as Checkpoint
32
33
 
33
34
  DEFAULT_NAME: str
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.079497 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.461351 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.056039 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.424371 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow
12
11
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
12
+ import metaflow
13
13
 
14
14
  from ..datastructures import CheckpointArtifact as CheckpointArtifact
15
15
  from .constructors import load_checkpoint as load_checkpoint
@@ -63,6 +63,8 @@ class Checkpoint(object, metaclass=type):
63
63
  If `files`, saves directory directly to the datastore.
64
64
  """
65
65
  ...
66
+ def generate_key(self, name: str, version_id: int = None):
67
+ ...
66
68
  def __enter__(self):
67
69
  ...
68
70
  def __exit__(self, exc_type, exc_val, exc_tb):
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.110142 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.493638 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,6 +1,6 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.080084 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.427328 #
5
5
  ######################################################################################################
6
6
 
@@ -0,0 +1,95 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.427403 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.context
13
+ import metaflow.exception
14
+ import metaflow
15
+
16
+ from ......exception import MetaflowException as MetaflowException
17
+
18
+ TYPE_CHECKING: bool
19
+
20
+ ARTIFACT_STORE_CONFIG_ENV_VAR: str
21
+
22
+ def will_switch_context(func):
23
+ """
24
+ A decorator that wraps methods in ArtifactStoreContext.
25
+ It takes the return value of the decorated method and passes it to
26
+ self.switch_context(), then returns None.
27
+ """
28
+ ...
29
+
30
+ class DatastoreContextValidation(tuple, metaclass=type):
31
+ """
32
+ DatastoreContextInfo(is_valid, needs_external_context, context_mismatch)
33
+ """
34
+ @staticmethod
35
+ def __new__(_cls, is_valid, needs_external_context, context_mismatch):
36
+ """
37
+ Create new instance of DatastoreContextInfo(is_valid, needs_external_context, context_mismatch)
38
+ """
39
+ ...
40
+ def __repr__(self):
41
+ """
42
+ Return a nicely formatted representation string
43
+ """
44
+ ...
45
+ def __getnewargs__(self):
46
+ """
47
+ Return self as a plain tuple. Used by copy and pickle.
48
+ """
49
+ ...
50
+ ...
51
+
52
+ class UnresolvableDatastoreException(metaflow.exception.MetaflowException, metaclass=type):
53
+ ...
54
+
55
+ class ArtifactStoreContext(object, metaclass=type):
56
+ """
57
+ This class will act as a singleton to switch the datastore so that
58
+ Any checkpoint operations will be done in the correct datastore. This
59
+ can be a useful way of short-circuting a datastore switch between runtime
60
+ And post-runtime retrieval operations.
61
+ """
62
+ @property
63
+ def MF_DATASTORES(self):
64
+ ...
65
+ def __init__(self):
66
+ ...
67
+ def flow_init_context(self, *args, **kwargs):
68
+ ...
69
+ def switch_context(self, context):
70
+ ...
71
+ def get(self):
72
+ ...
73
+ def context_from_run(self, *args, **kwargs):
74
+ ...
75
+ def context_from_task(self, *args, **kwargs):
76
+ ...
77
+ def current_context_matches_task_metadata(self, task: "metaflow.Task"):
78
+ ...
79
+ def to_task_metadata(self):
80
+ ...
81
+ def default(self):
82
+ ...
83
+ ...
84
+
85
+ datastore_context: ArtifactStoreContext
86
+
87
+ def artifact_store_from(run = None, task = None, config = None):
88
+ """
89
+ This context manager can be used to switch the artifact store
90
+ for a block of code. This is useful when users maybe accessing
91
+ checkpoints/models from a different datastore using the
92
+ `@with_artifact_store` decorator.
93
+ """
94
+ ...
95
+
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.107134 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.495660 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core
12
11
  import metaflow.datastore.datastore_storage
12
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core
13
13
 
14
14
  from ......exception import MetaflowException as MetaflowException
15
15
  from ..utils.tar_utils import create_tarball_on_disk as create_tarball_on_disk
@@ -69,6 +69,8 @@ class ListPathResult(tuple, metaclass=type):
69
69
 
70
70
  COMPRESSION_METHOD: None
71
71
 
72
+ COMPATIBLE_BACKENDS: list
73
+
72
74
  class STORAGE_FORMATS(object, metaclass=type):
73
75
  ...
74
76
 
@@ -105,9 +107,6 @@ class ObjectStorage(object, metaclass=type):
105
107
  ...
106
108
  def set_full_prefix(self, root_prefix):
107
109
  ...
108
- @staticmethod
109
- def resolve_root(storage_type):
110
- ...
111
110
  def full_base_url(self, prefix = None):
112
111
  ...
113
112
  def create_key_name(self, *args):
@@ -142,6 +141,8 @@ class ObjectStorage(object, metaclass=type):
142
141
  List all objects in the datastore's `keys` index.
143
142
  """
144
143
  ...
144
+ def __str__(self) -> str:
145
+ ...
145
146
  ...
146
147
 
147
148
  class DatastoreInterface(object, metaclass=type):
@@ -183,6 +184,3 @@ class DatastoreInterface(object, metaclass=type):
183
184
  ...
184
185
  ...
185
186
 
186
- def resolve_root(datastore_type):
187
- ...
188
-
@@ -0,0 +1,135 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.494008 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.decorators
13
+
14
+ from ......metadata_provider.metadata import MetaDatum as MetaDatum
15
+ from ......exception import MetaflowException as MetaflowException
16
+
17
+ class ArtifactStoreFlowDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
18
+ """
19
+ Allows setting external datastores to save data for the
20
+ `@checkpoint`/`@model`/`@huggingface_hub` decorators.
21
+
22
+ This decorator is useful when users wish to save data to a different datastore
23
+ than what is configured in Metaflow. This can be for variety of reasons:
24
+
25
+ 1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
26
+ 2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
27
+ - Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
28
+ 3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
29
+ - Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
30
+
31
+ Usage:
32
+ ----------
33
+
34
+ - Using a custom IAM role to access the datastore.
35
+
36
+ ```python
37
+ @with_artifact_store(
38
+ type="s3",
39
+ config=lambda: {
40
+ "root": "s3://my-bucket-foo/path/to/root",
41
+ "role_arn": ROLE,
42
+ },
43
+ )
44
+ class MyFlow(FlowSpec):
45
+
46
+ @checkpoint
47
+ @step
48
+ def start(self):
49
+ with open("my_file.txt", "w") as f:
50
+ f.write("Hello, World!")
51
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
52
+ self.next(self.end)
53
+
54
+ ```
55
+
56
+ - Using credentials to access the s3-compatible datastore.
57
+
58
+ ```python
59
+ @with_artifact_store(
60
+ type="s3",
61
+ config=lambda: {
62
+ "root": "s3://my-bucket-foo/path/to/root",
63
+ "client_params": {
64
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
65
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
66
+ },
67
+ },
68
+ )
69
+ class MyFlow(FlowSpec):
70
+
71
+ @checkpoint
72
+ @step
73
+ def start(self):
74
+ with open("my_file.txt", "w") as f:
75
+ f.write("Hello, World!")
76
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
77
+ self.next(self.end)
78
+
79
+ ```
80
+
81
+ - Accessing objects stored in external datastores after task execution.
82
+
83
+ ```python
84
+ run = Run("CheckpointsTestsFlow/8992")
85
+ with artifact_store_from(run=run, config={
86
+ "client_params": {
87
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
88
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
89
+ },
90
+ }):
91
+ with Checkpoint() as cp:
92
+ latest = cp.list(
93
+ task=run["start"].task
94
+ )[0]
95
+ print(latest)
96
+ cp.load(
97
+ latest,
98
+ "test-checkpoints"
99
+ )
100
+
101
+ task = Task("TorchTuneFlow/8484/train/53673")
102
+ with artifact_store_from(run=run, config={
103
+ "client_params": {
104
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
105
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
106
+ },
107
+ }):
108
+ load_model(
109
+ task.data.model_ref,
110
+ "test-models"
111
+ )
112
+ ```
113
+ Parameters:
114
+ ----------
115
+
116
+ type: str
117
+ The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
118
+
119
+ config: dict or Callable
120
+ Dictionary of configuration options for the datastore. The following keys are required:
121
+ - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
122
+ - example: 's3://bucket-name/path/to/root'
123
+ - example: 'gs://bucket-name/path/to/root'
124
+ - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
125
+ - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
126
+ - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
127
+ - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
128
+ """
129
+ def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options):
130
+ ...
131
+ ...
132
+
133
+ def set_datastore_context(flow, metadata, run_id, step_name, task_id, retry_count):
134
+ ...
135
+
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.162713 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.550134 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,19 +1,20 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.080131 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.462042 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import metaflow
10
9
  import typing
10
+ import metaflow
11
11
  if typing.TYPE_CHECKING:
12
- import metaflow
13
12
  import metaflow.exception
13
+ import metaflow
14
14
 
15
+ from ......metaflow_current import current as current
15
16
  from ......exception import MetaflowException as MetaflowException
16
- from .core import resolve_root as resolve_root
17
+ from .context import datastore_context as datastore_context
17
18
 
18
19
  TYPE_CHECKING: bool
19
20
 
@@ -24,5 +25,32 @@ def init_datastorage_object():
24
25
  ...
25
26
 
26
27
  def resolve_storage_backend(pathspec: typing.Union[str, "metaflow.Task"] = None):
28
+ """
29
+ This function is ONLY called when the user is calling `Checkpoint.list`.
30
+ What happens when users call list:
31
+ 1. The task has completed so the list call is using the `_task_checkpoints` data-artifact to list the checkpoints.
32
+ 2. The task is still running or has crashed. This means that in order to list the checkpoints, we need access to the datastore.
33
+
34
+ For case `1`, this code path wont even be called because the data-artifact is already a separate object.
35
+ For case `2` is where this code path is important. But since we now expose a `artifact_store_from` context manager, we
36
+ know pre-hand what the datastore needs to be and the datastore context has already been switched.
37
+
38
+ SO in turn, one can make the argument that this function is not needed and we can just have a
39
+ users do a `Checkpoint.list` under the context manager if they need to access objects in a different datastore.
40
+
41
+ This is a respectible pattern since directly reading the task metadata and trying to do a list call is not a good pattern
42
+ since the creds of metaflow default datastore might not be the same as the datastore the user wants to access.
43
+
44
+ There is a function that can help verify if the datastore set in the task metadata
45
+ is the same as the default datastore. If it is not, then we should shout warning messages
46
+ to the user. In case the permissions are the same, nothing wrong happens, if they are not then
47
+ user will have some hint in the logs to help them figure out the issue.
48
+ """
49
+ ...
50
+
51
+ class FlowNotRunningException(metaflow.exception.MetaflowException, metaclass=type):
52
+ ...
53
+
54
+ def storage_backend_from_flow(flow: "metaflow.FlowSpec"):
27
55
  ...
28
56
 
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.111222 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.496778 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,13 +1,13 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.057191 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.426199 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import metaflow
10
9
  import typing
10
+ import metaflow
11
11
  if typing.TYPE_CHECKING:
12
12
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
13
 
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.079809 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.461664 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,6 +1,6 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.082165 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.464125 #
5
5
  ######################################################################################################
6
6
 
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.082218 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.464177 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,8 +9,8 @@ from __future__ import annotations
9
9
  import metaflow
10
10
  import typing
11
11
  if typing.TYPE_CHECKING:
12
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator
13
12
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
13
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator
14
14
 
15
15
  from ..checkpoints.decorator import CheckpointDecorator as CheckpointDecorator
16
16
  from ..checkpoints.decorator import CurrentCheckpointer as CurrentCheckpointer
@@ -1,6 +1,6 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-16T21:50:24.083117 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.1);ob(v1) #
4
+ # Generated on 2025-04-17T01:41:06.465095 #
5
5
  ######################################################################################################
6
6