ob-metaflow-stubs 6.0.3.154__py2.py3-none-any.whl → 6.0.3.155rc0__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (200) hide show
  1. metaflow-stubs/__init__.pyi +800 -685
  2. metaflow-stubs/cards.pyi +2 -2
  3. metaflow-stubs/cli.pyi +2 -2
  4. metaflow-stubs/cli_components/__init__.pyi +2 -2
  5. metaflow-stubs/cli_components/utils.pyi +2 -2
  6. metaflow-stubs/client/__init__.pyi +2 -2
  7. metaflow-stubs/client/core.pyi +6 -6
  8. metaflow-stubs/client/filecache.pyi +3 -3
  9. metaflow-stubs/events.pyi +2 -2
  10. metaflow-stubs/exception.pyi +2 -2
  11. metaflow-stubs/flowspec.pyi +4 -4
  12. metaflow-stubs/generated_for.txt +1 -1
  13. metaflow-stubs/includefile.pyi +5 -5
  14. metaflow-stubs/info_file.pyi +2 -2
  15. metaflow-stubs/metadata_provider/__init__.pyi +2 -2
  16. metaflow-stubs/metadata_provider/heartbeat.pyi +2 -2
  17. metaflow-stubs/metadata_provider/metadata.pyi +3 -3
  18. metaflow-stubs/metadata_provider/util.pyi +2 -2
  19. metaflow-stubs/metaflow_config.pyi +2 -2
  20. metaflow-stubs/metaflow_current.pyi +96 -96
  21. metaflow-stubs/mf_extensions/__init__.pyi +2 -2
  22. metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi +2 -2
  23. metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi +2 -2
  24. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi +2 -2
  25. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/__init__.pyi +2 -2
  26. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/async_cards.pyi +2 -2
  27. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/deco_injection_mixin.pyi +2 -2
  28. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/card_utils/extra_components.pyi +4 -4
  29. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi +2 -2
  30. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/__init__.pyi +2 -2
  31. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/checkpoint_lister.pyi +5 -5
  32. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/cards/lineage_card.pyi +2 -2
  33. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi +7 -5
  34. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi +2 -2
  35. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/core.pyi +11 -5
  36. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/decorator.pyi +7 -6
  37. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/exceptions.pyi +2 -2
  38. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/final_api.pyi +5 -3
  39. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/lineage.pyi +2 -2
  40. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/__init__.pyi +2 -2
  41. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/context.pyi +95 -0
  42. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/core.pyi +4 -8
  43. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/decorator.pyi +135 -0
  44. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/exceptions.pyi +2 -2
  45. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/task_utils.pyi +32 -4
  46. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastore/utils.pyi +2 -2
  47. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/datastructures.pyi +3 -3
  48. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi +2 -2
  49. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/__init__.pyi +2 -2
  50. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/hf_hub/decorator.pyi +3 -3
  51. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/__init__.pyi +2 -2
  52. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/core.pyi +3 -3
  53. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/exceptions.pyi +2 -2
  54. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/modeling_utils/model_storage.pyi +4 -4
  55. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi +2 -2
  56. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi +2 -2
  57. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi +2 -2
  58. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi +2 -2
  59. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/__init__.pyi +2 -2
  60. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/base.pyi +2 -2
  61. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/serialization_handler/tar.pyi +3 -3
  62. metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi +3 -3
  63. metaflow-stubs/mf_extensions/outerbounds/__init__.pyi +2 -2
  64. metaflow-stubs/mf_extensions/outerbounds/plugins/__init__.pyi +2 -2
  65. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/__init__.pyi +2 -2
  66. metaflow-stubs/mf_extensions/outerbounds/plugins/card_utilities/injector.pyi +2 -2
  67. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/__init__.pyi +2 -2
  68. metaflow-stubs/mf_extensions/outerbounds/plugins/ollama/ollama.pyi +2 -2
  69. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/__init__.pyi +2 -2
  70. metaflow-stubs/mf_extensions/outerbounds/plugins/snowflake/snowflake.pyi +2 -2
  71. metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi +2 -2
  72. metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi +2 -2
  73. metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi +2 -2
  74. metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi +2 -2
  75. metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi +2 -2
  76. metaflow-stubs/multicore_utils.pyi +2 -2
  77. metaflow-stubs/parameters.pyi +5 -5
  78. metaflow-stubs/plugins/__init__.pyi +9 -9
  79. metaflow-stubs/plugins/airflow/__init__.pyi +2 -2
  80. metaflow-stubs/plugins/airflow/airflow_utils.pyi +2 -2
  81. metaflow-stubs/plugins/airflow/exception.pyi +2 -2
  82. metaflow-stubs/plugins/airflow/sensors/__init__.pyi +2 -2
  83. metaflow-stubs/plugins/airflow/sensors/base_sensor.pyi +2 -2
  84. metaflow-stubs/plugins/airflow/sensors/external_task_sensor.pyi +2 -2
  85. metaflow-stubs/plugins/airflow/sensors/s3_sensor.pyi +2 -2
  86. metaflow-stubs/plugins/argo/__init__.pyi +2 -2
  87. metaflow-stubs/plugins/argo/argo_client.pyi +2 -2
  88. metaflow-stubs/plugins/argo/argo_events.pyi +2 -2
  89. metaflow-stubs/plugins/argo/argo_workflows.pyi +4 -4
  90. metaflow-stubs/plugins/argo/argo_workflows_decorator.pyi +3 -3
  91. metaflow-stubs/plugins/argo/argo_workflows_deployer.pyi +5 -5
  92. metaflow-stubs/plugins/argo/argo_workflows_deployer_objects.pyi +3 -3
  93. metaflow-stubs/plugins/aws/__init__.pyi +2 -2
  94. metaflow-stubs/plugins/aws/aws_client.pyi +2 -2
  95. metaflow-stubs/plugins/aws/aws_utils.pyi +2 -2
  96. metaflow-stubs/plugins/aws/batch/__init__.pyi +2 -2
  97. metaflow-stubs/plugins/aws/batch/batch.pyi +2 -2
  98. metaflow-stubs/plugins/aws/batch/batch_client.pyi +2 -2
  99. metaflow-stubs/plugins/aws/batch/batch_decorator.pyi +2 -2
  100. metaflow-stubs/plugins/aws/secrets_manager/__init__.pyi +2 -2
  101. metaflow-stubs/plugins/aws/secrets_manager/aws_secrets_manager_secrets_provider.pyi +3 -3
  102. metaflow-stubs/plugins/aws/step_functions/__init__.pyi +2 -2
  103. metaflow-stubs/plugins/aws/step_functions/event_bridge_client.pyi +2 -2
  104. metaflow-stubs/plugins/aws/step_functions/schedule_decorator.pyi +2 -2
  105. metaflow-stubs/plugins/aws/step_functions/step_functions.pyi +2 -2
  106. metaflow-stubs/plugins/aws/step_functions/step_functions_client.pyi +2 -2
  107. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer.pyi +5 -5
  108. metaflow-stubs/plugins/aws/step_functions/step_functions_deployer_objects.pyi +4 -4
  109. metaflow-stubs/plugins/azure/__init__.pyi +2 -2
  110. metaflow-stubs/plugins/azure/azure_credential.pyi +2 -2
  111. metaflow-stubs/plugins/azure/azure_exceptions.pyi +2 -2
  112. metaflow-stubs/plugins/azure/azure_secret_manager_secrets_provider.pyi +3 -3
  113. metaflow-stubs/plugins/azure/azure_utils.pyi +2 -2
  114. metaflow-stubs/plugins/azure/blob_service_client_factory.pyi +2 -2
  115. metaflow-stubs/plugins/azure/includefile_support.pyi +2 -2
  116. metaflow-stubs/plugins/cards/__init__.pyi +2 -2
  117. metaflow-stubs/plugins/cards/card_client.pyi +3 -3
  118. metaflow-stubs/plugins/cards/card_creator.pyi +2 -2
  119. metaflow-stubs/plugins/cards/card_datastore.pyi +2 -2
  120. metaflow-stubs/plugins/cards/card_decorator.pyi +2 -2
  121. metaflow-stubs/plugins/cards/card_modules/__init__.pyi +2 -2
  122. metaflow-stubs/plugins/cards/card_modules/basic.pyi +3 -3
  123. metaflow-stubs/plugins/cards/card_modules/card.pyi +2 -2
  124. metaflow-stubs/plugins/cards/card_modules/components.pyi +5 -5
  125. metaflow-stubs/plugins/cards/card_modules/convert_to_native_type.pyi +2 -2
  126. metaflow-stubs/plugins/cards/card_modules/renderer_tools.pyi +2 -2
  127. metaflow-stubs/plugins/cards/card_modules/test_cards.pyi +2 -2
  128. metaflow-stubs/plugins/cards/card_resolver.pyi +2 -2
  129. metaflow-stubs/plugins/cards/component_serializer.pyi +2 -2
  130. metaflow-stubs/plugins/cards/exception.pyi +2 -2
  131. metaflow-stubs/plugins/catch_decorator.pyi +3 -3
  132. metaflow-stubs/plugins/datatools/__init__.pyi +2 -2
  133. metaflow-stubs/plugins/datatools/local.pyi +2 -2
  134. metaflow-stubs/plugins/datatools/s3/__init__.pyi +2 -2
  135. metaflow-stubs/plugins/datatools/s3/s3.pyi +5 -5
  136. metaflow-stubs/plugins/datatools/s3/s3tail.pyi +2 -2
  137. metaflow-stubs/plugins/datatools/s3/s3util.pyi +2 -2
  138. metaflow-stubs/plugins/debug_logger.pyi +2 -2
  139. metaflow-stubs/plugins/debug_monitor.pyi +2 -2
  140. metaflow-stubs/plugins/environment_decorator.pyi +2 -2
  141. metaflow-stubs/plugins/events_decorator.pyi +2 -2
  142. metaflow-stubs/plugins/frameworks/__init__.pyi +2 -2
  143. metaflow-stubs/plugins/frameworks/pytorch.pyi +2 -2
  144. metaflow-stubs/plugins/gcp/__init__.pyi +2 -2
  145. metaflow-stubs/plugins/gcp/gcp_secret_manager_secrets_provider.pyi +3 -3
  146. metaflow-stubs/plugins/gcp/gs_exceptions.pyi +2 -2
  147. metaflow-stubs/plugins/gcp/gs_storage_client_factory.pyi +2 -2
  148. metaflow-stubs/plugins/gcp/gs_utils.pyi +2 -2
  149. metaflow-stubs/plugins/gcp/includefile_support.pyi +2 -2
  150. metaflow-stubs/plugins/kubernetes/__init__.pyi +2 -2
  151. metaflow-stubs/plugins/kubernetes/kube_utils.pyi +3 -3
  152. metaflow-stubs/plugins/kubernetes/kubernetes.pyi +2 -2
  153. metaflow-stubs/plugins/kubernetes/kubernetes_client.pyi +2 -2
  154. metaflow-stubs/plugins/kubernetes/kubernetes_decorator.pyi +2 -2
  155. metaflow-stubs/plugins/kubernetes/kubernetes_jobsets.pyi +2 -2
  156. metaflow-stubs/plugins/kubernetes/spot_monitor_sidecar.pyi +2 -2
  157. metaflow-stubs/plugins/ollama/__init__.pyi +3 -3
  158. metaflow-stubs/plugins/parallel_decorator.pyi +2 -2
  159. metaflow-stubs/plugins/perimeters.pyi +2 -2
  160. metaflow-stubs/plugins/project_decorator.pyi +2 -2
  161. metaflow-stubs/plugins/pypi/__init__.pyi +3 -3
  162. metaflow-stubs/plugins/pypi/conda_decorator.pyi +2 -2
  163. metaflow-stubs/plugins/pypi/conda_environment.pyi +5 -5
  164. metaflow-stubs/plugins/pypi/parsers.pyi +2 -2
  165. metaflow-stubs/plugins/pypi/pypi_decorator.pyi +2 -2
  166. metaflow-stubs/plugins/pypi/pypi_environment.pyi +2 -2
  167. metaflow-stubs/plugins/pypi/utils.pyi +2 -2
  168. metaflow-stubs/plugins/resources_decorator.pyi +2 -2
  169. metaflow-stubs/plugins/retry_decorator.pyi +2 -2
  170. metaflow-stubs/plugins/secrets/__init__.pyi +3 -3
  171. metaflow-stubs/plugins/secrets/inline_secrets_provider.pyi +3 -3
  172. metaflow-stubs/plugins/secrets/secrets_decorator.pyi +2 -2
  173. metaflow-stubs/plugins/snowflake/__init__.pyi +2 -2
  174. metaflow-stubs/plugins/storage_executor.pyi +2 -2
  175. metaflow-stubs/plugins/test_unbounded_foreach_decorator.pyi +2 -2
  176. metaflow-stubs/plugins/timeout_decorator.pyi +3 -3
  177. metaflow-stubs/profilers/__init__.pyi +2 -2
  178. metaflow-stubs/pylint_wrapper.pyi +2 -2
  179. metaflow-stubs/runner/__init__.pyi +2 -2
  180. metaflow-stubs/runner/deployer.pyi +30 -30
  181. metaflow-stubs/runner/deployer_impl.pyi +3 -3
  182. metaflow-stubs/runner/metaflow_runner.pyi +4 -4
  183. metaflow-stubs/runner/nbdeploy.pyi +2 -2
  184. metaflow-stubs/runner/nbrun.pyi +2 -2
  185. metaflow-stubs/runner/subprocess_manager.pyi +2 -2
  186. metaflow-stubs/runner/utils.pyi +3 -3
  187. metaflow-stubs/system/__init__.pyi +2 -2
  188. metaflow-stubs/system/system_logger.pyi +3 -3
  189. metaflow-stubs/system/system_monitor.pyi +2 -2
  190. metaflow-stubs/tagging_util.pyi +2 -2
  191. metaflow-stubs/tuple_util.pyi +2 -2
  192. metaflow-stubs/user_configs/__init__.pyi +2 -2
  193. metaflow-stubs/user_configs/config_decorators.pyi +6 -6
  194. metaflow-stubs/user_configs/config_options.pyi +4 -4
  195. metaflow-stubs/user_configs/config_parameters.pyi +6 -6
  196. {ob_metaflow_stubs-6.0.3.154.dist-info → ob_metaflow_stubs-6.0.3.155rc0.dist-info}/METADATA +1 -1
  197. ob_metaflow_stubs-6.0.3.155rc0.dist-info/RECORD +200 -0
  198. ob_metaflow_stubs-6.0.3.154.dist-info/RECORD +0 -198
  199. {ob_metaflow_stubs-6.0.3.154.dist-info → ob_metaflow_stubs-6.0.3.155rc0.dist-info}/WHEEL +0 -0
  200. {ob_metaflow_stubs-6.0.3.154.dist-info → ob_metaflow_stubs-6.0.3.155rc0.dist-info}/top_level.txt +0 -0
@@ -1,18 +1,18 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.946819 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.453263 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
- import threading
11
9
  import metaflow
10
+ import threading
11
+ import typing
12
12
  if typing.TYPE_CHECKING:
13
+ import threading
13
14
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.card_utils.async_cards
14
15
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
15
- import threading
16
16
 
17
17
  from ...card_utils.deco_injection_mixin import CardDecoratorInjector as CardDecoratorInjector
18
18
  from ...card_utils.async_cards import CardRefresher as CardRefresher
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.947254 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.453861 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,18 +1,18 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.983812 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.521318 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
9
  import metaflow
10
+ import typing
11
11
  if typing.TYPE_CHECKING:
12
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
12
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
13
13
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core
14
14
  import metaflow.datastore.datastore_storage
15
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
15
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
16
16
 
17
17
  from ..exceptions import KeyNotCompatibleWithObjectException as KeyNotCompatibleWithObjectException
18
18
  from ..utils.identity_utils import pathspec_hash as pathspec_hash
@@ -123,6 +123,8 @@ class CheckpointDatastore(metaflow.mf_extensions.obcheckpoint.plugins.machine_le
123
123
  @classmethod
124
124
  def decompose_key(cls, key) -> CheckpointsPathComponents:
125
125
  ...
126
+ def __str__(self):
127
+ ...
126
128
  def __init__(self, artifact_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, metadata_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, artifact_metadatastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage):
127
129
  ...
128
130
  ...
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.916409 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.414761 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,17 +1,17 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.943251 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.449162 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
12
+ import metaflow
11
13
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.core
12
14
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
- import metaflow
14
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
15
15
 
16
16
  from ......metaflow_current import current as current
17
17
  from ......exception import MetaflowException as MetaflowException
@@ -22,6 +22,7 @@ from .checkpoint_storage import CheckpointDatastore as CheckpointDatastore
22
22
  from ..datastructures import CheckpointArtifact as CheckpointArtifact
23
23
  from ..datastore.task_utils import init_datastorage_object as init_datastorage_object
24
24
  from ..datastore.task_utils import resolve_storage_backend as resolve_task_storage_backend
25
+ from ..datastore.task_utils import storage_backend_from_flow as storage_backend_from_flow
25
26
 
26
27
  TYPE_CHECKING: bool
27
28
 
@@ -45,6 +46,8 @@ class Checkpointer(object, metaclass=type):
45
46
  ...
46
47
  def load_metadata(self, version_id: int = None, name = 'mfchckpt') -> dict:
47
48
  ...
49
+ def artifact_id(self, name: str, version_id: int = None):
50
+ ...
48
51
  def save(self, path: str, metadata = {}, latest = True, name = 'mfchckpt', storage_format = 'files') -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact:
49
52
  ...
50
53
  ...
@@ -88,6 +91,9 @@ class ScopeResolver(object, metaclass=type):
88
91
  ...
89
92
  ...
90
93
 
94
+ def warning_message(message, logger = None, ts = False, prefix = '[@checkpoint][warning]'):
95
+ ...
96
+
91
97
  class ReadResolver(object, metaclass=type):
92
98
  """
93
99
  Responsible for instantiating the `CheckpointDatastore` during read operations
@@ -102,7 +108,7 @@ class ReadResolver(object, metaclass=type):
102
108
  """
103
109
  ...
104
110
  @classmethod
105
- def from_key_and_pathspec(cls, pathspec, checkpoint_key):
111
+ def from_checkpoint(cls, checkpoint: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact):
106
112
  """
107
113
  """
108
114
  ...
@@ -1,18 +1,18 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.918537 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.416891 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
9
  import metaflow
10
+ import typing
11
11
  if typing.TYPE_CHECKING:
12
- import os
13
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
14
- import metaflow.decorators
15
12
  import metaflow
13
+ import metaflow.decorators
14
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
15
+ import os
16
16
 
17
17
  from .exceptions import CheckpointException as CheckpointException
18
18
  from ..utils import flowspec_utils as flowspec_utils
@@ -28,6 +28,7 @@ from .constructors import load_checkpoint as load_checkpoint
28
28
  from .core import ScopeResolver as ScopeResolver
29
29
  from .core import CheckpointLoadPolicy as CheckpointLoadPolicy
30
30
  from ..datastructures import CheckpointArtifact as CheckpointArtifact
31
+ from ..datastore.decorator import set_datastore_context as set_datastore_context
31
32
  from .final_api import Checkpoint as Checkpoint
32
33
 
33
34
  DEFAULT_NAME: str
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.916640 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.414980 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,15 +1,15 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.893051 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.390393 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
9
  import typing
10
10
  if typing.TYPE_CHECKING:
11
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
12
11
  import metaflow
12
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
13
 
14
14
  from ..datastructures import CheckpointArtifact as CheckpointArtifact
15
15
  from .constructors import load_checkpoint as load_checkpoint
@@ -63,6 +63,8 @@ class Checkpoint(object, metaclass=type):
63
63
  If `files`, saves directory directly to the datastore.
64
64
  """
65
65
  ...
66
+ def generate_key(self, name: str, version_id: int = None):
67
+ ...
66
68
  def __enter__(self):
67
69
  ...
68
70
  def __exit__(self, exc_type, exc_val, exc_tb):
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.947530 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.454251 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,6 +1,6 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.917233 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.392105 #
5
5
  ######################################################################################################
6
6
 
@@ -0,0 +1,95 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.392151 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.context
13
+ import metaflow.exception
14
+ import metaflow
15
+
16
+ from ......exception import MetaflowException as MetaflowException
17
+
18
+ TYPE_CHECKING: bool
19
+
20
+ ARTIFACT_STORE_CONFIG_ENV_VAR: str
21
+
22
+ def will_switch_context(func):
23
+ """
24
+ A decorator that wraps methods in ArtifactStoreContext.
25
+ It takes the return value of the decorated method and passes it to
26
+ self.switch_context(), then returns None.
27
+ """
28
+ ...
29
+
30
+ class DatastoreContextValidation(tuple, metaclass=type):
31
+ """
32
+ DatastoreContextInfo(is_valid, needs_external_context, context_mismatch)
33
+ """
34
+ @staticmethod
35
+ def __new__(_cls, is_valid, needs_external_context, context_mismatch):
36
+ """
37
+ Create new instance of DatastoreContextInfo(is_valid, needs_external_context, context_mismatch)
38
+ """
39
+ ...
40
+ def __repr__(self):
41
+ """
42
+ Return a nicely formatted representation string
43
+ """
44
+ ...
45
+ def __getnewargs__(self):
46
+ """
47
+ Return self as a plain tuple. Used by copy and pickle.
48
+ """
49
+ ...
50
+ ...
51
+
52
+ class UnresolvableDatastoreException(metaflow.exception.MetaflowException, metaclass=type):
53
+ ...
54
+
55
+ class ArtifactStoreContext(object, metaclass=type):
56
+ """
57
+ This class will act as a singleton to switch the datastore so that
58
+ Any checkpoint operations will be done in the correct datastore. This
59
+ can be a useful way of short-circuting a datastore switch between runtime
60
+ And post-runtime retrieval operations.
61
+ """
62
+ @property
63
+ def MF_DATASTORES(self):
64
+ ...
65
+ def __init__(self):
66
+ ...
67
+ def flow_init_context(self, *args, **kwargs):
68
+ ...
69
+ def switch_context(self, context):
70
+ ...
71
+ def get(self):
72
+ ...
73
+ def context_from_run(self, *args, **kwargs):
74
+ ...
75
+ def context_from_task(self, *args, **kwargs):
76
+ ...
77
+ def current_context_matches_task_metadata(self, task: "metaflow.Task"):
78
+ ...
79
+ def to_task_metadata(self):
80
+ ...
81
+ def default(self):
82
+ ...
83
+ ...
84
+
85
+ datastore_context: ArtifactStoreContext
86
+
87
+ def artifact_store_from(run = None, task = None, config = None):
88
+ """
89
+ This context manager can be used to switch the artifact store
90
+ for a block of code. This is useful when users maybe accessing
91
+ checkpoints/models from a different datastore using the
92
+ `@with_artifact_store` decorator.
93
+ """
94
+ ...
95
+
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.944494 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.457016 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -105,9 +105,6 @@ class ObjectStorage(object, metaclass=type):
105
105
  ...
106
106
  def set_full_prefix(self, root_prefix):
107
107
  ...
108
- @staticmethod
109
- def resolve_root(storage_type):
110
- ...
111
108
  def full_base_url(self, prefix = None):
112
109
  ...
113
110
  def create_key_name(self, *args):
@@ -142,6 +139,8 @@ class ObjectStorage(object, metaclass=type):
142
139
  List all objects in the datastore's `keys` index.
143
140
  """
144
141
  ...
142
+ def __str__(self) -> str:
143
+ ...
145
144
  ...
146
145
 
147
146
  class DatastoreInterface(object, metaclass=type):
@@ -183,6 +182,3 @@ class DatastoreInterface(object, metaclass=type):
183
182
  ...
184
183
  ...
185
184
 
186
- def resolve_root(datastore_type):
187
- ...
188
-
@@ -0,0 +1,135 @@
1
+ ######################################################################################################
2
+ # Auto-generated Metaflow stub file #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.454787 #
5
+ ######################################################################################################
6
+
7
+ from __future__ import annotations
8
+
9
+ import metaflow
10
+ import typing
11
+ if typing.TYPE_CHECKING:
12
+ import metaflow.decorators
13
+
14
+ from ......metadata_provider.metadata import MetaDatum as MetaDatum
15
+ from ......exception import MetaflowException as MetaflowException
16
+
17
+ class ArtifactStoreFlowDecorator(metaflow.decorators.FlowDecorator, metaclass=type):
18
+ """
19
+ Allows setting external datastores to save data for the
20
+ `@checkpoint`/`@model`/`@huggingface_hub` decorators.
21
+
22
+ This decorator is useful when users wish to save data to a different datastore
23
+ than what is configured in Metaflow. This can be for variety of reasons:
24
+
25
+ 1. Data security: The objects needs to be stored in a bucket (object storage) that is not accessible by other flows.
26
+ 2. Data Locality: The location where the task is executing is not located in the same region as the datastore.
27
+ - Example: Metaflow datastore lives in US East, but the task is executing in Finland datacenters.
28
+ 3. Data Lifecycle Policies: The objects need to be archived / managed separately from the Metaflow managed objects.
29
+ - Example: Flow is training very large models that need to be stored separately and will be deleted more aggressively than the Metaflow managed objects.
30
+
31
+ Usage:
32
+ ----------
33
+
34
+ - Using a custom IAM role to access the datastore.
35
+
36
+ ```python
37
+ @with_artifact_store(
38
+ type="s3",
39
+ config=lambda: {
40
+ "root": "s3://my-bucket-foo/path/to/root",
41
+ "role_arn": ROLE,
42
+ },
43
+ )
44
+ class MyFlow(FlowSpec):
45
+
46
+ @checkpoint
47
+ @step
48
+ def start(self):
49
+ with open("my_file.txt", "w") as f:
50
+ f.write("Hello, World!")
51
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
52
+ self.next(self.end)
53
+
54
+ ```
55
+
56
+ - Using credentials to access the s3-compatible datastore.
57
+
58
+ ```python
59
+ @with_artifact_store(
60
+ type="s3",
61
+ config=lambda: {
62
+ "root": "s3://my-bucket-foo/path/to/root",
63
+ "client_params": {
64
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
65
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
66
+ },
67
+ },
68
+ )
69
+ class MyFlow(FlowSpec):
70
+
71
+ @checkpoint
72
+ @step
73
+ def start(self):
74
+ with open("my_file.txt", "w") as f:
75
+ f.write("Hello, World!")
76
+ self.external_bucket_checkpoint = current.checkpoint.save("my_file.txt")
77
+ self.next(self.end)
78
+
79
+ ```
80
+
81
+ - Accessing objects stored in external datastores after task execution.
82
+
83
+ ```python
84
+ run = Run("CheckpointsTestsFlow/8992")
85
+ with artifact_store_from(run=run, config={
86
+ "client_params": {
87
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
88
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
89
+ },
90
+ }):
91
+ with Checkpoint() as cp:
92
+ latest = cp.list(
93
+ task=run["start"].task
94
+ )[0]
95
+ print(latest)
96
+ cp.load(
97
+ latest,
98
+ "test-checkpoints"
99
+ )
100
+
101
+ task = Task("TorchTuneFlow/8484/train/53673")
102
+ with artifact_store_from(run=run, config={
103
+ "client_params": {
104
+ "aws_access_key_id": os.environ.get("MY_CUSTOM_ACCESS_KEY"),
105
+ "aws_secret_access_key": os.environ.get("MY_CUSTOM_SECRET_KEY"),
106
+ },
107
+ }):
108
+ load_model(
109
+ task.data.model_ref,
110
+ "test-models"
111
+ )
112
+ ```
113
+ Parameters:
114
+ ----------
115
+
116
+ type: str
117
+ The type of the datastore. Can be one of 's3', 'gcs', 'azure' or any other supported metaflow Datastore.
118
+
119
+ config: dict or Callable
120
+ Dictionary of configuration options for the datastore. The following keys are required:
121
+ - root: The root path in the datastore where the data will be saved. (needs to be in the format expected by the datastore)
122
+ - example: 's3://bucket-name/path/to/root'
123
+ - example: 'gs://bucket-name/path/to/root'
124
+ - example: 'https://myblockacc.blob.core.windows.net/metaflow/'
125
+ - role_arn (optional): AWS IAM role to access s3 bucket (only when `type` is 's3')
126
+ - session_vars (optional): AWS session variables to access s3 bucket (only when `type` is 's3')
127
+ - client_params (optional): AWS client parameters to access s3 bucket (only when `type` is 's3')
128
+ """
129
+ def flow_init(self, flow, graph, environment, flow_datastore, metadata, logger, echo, options):
130
+ ...
131
+ ...
132
+
133
+ def set_datastore_context(flow, metadata, run_id, step_name, task_id, retry_count):
134
+ ...
135
+
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.999085 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.542883 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,19 +1,20 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.917283 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.415634 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
9
  import metaflow
10
+ import typing
11
11
  if typing.TYPE_CHECKING:
12
12
  import metaflow.exception
13
13
  import metaflow
14
14
 
15
+ from ......metaflow_current import current as current
15
16
  from ......exception import MetaflowException as MetaflowException
16
- from .core import resolve_root as resolve_root
17
+ from .context import datastore_context as datastore_context
17
18
 
18
19
  TYPE_CHECKING: bool
19
20
 
@@ -24,5 +25,32 @@ def init_datastorage_object():
24
25
  ...
25
26
 
26
27
  def resolve_storage_backend(pathspec: typing.Union[str, "metaflow.Task"] = None):
28
+ """
29
+ This function is ONLY called when the user is calling `Checkpoint.list`.
30
+ What happens when users call list:
31
+ 1. The task has completed so the list call is using the `_task_checkpoints` data-artifact to list the checkpoints.
32
+ 2. The task is still running or has crashed. This means that in order to list the checkpoints, we need access to the datastore.
33
+
34
+ For case `1`, this code path wont even be called because the data-artifact is already a separate object.
35
+ For case `2` is where this code path is important. But since we now expose a `artifact_store_from` context manager, we
36
+ know pre-hand what the datastore needs to be and the datastore context has already been switched.
37
+
38
+ SO in turn, one can make the argument that this function is not needed and we can just have a
39
+ users do a `Checkpoint.list` under the context manager if they need to access objects in a different datastore.
40
+
41
+ This is a respectible pattern since directly reading the task metadata and trying to do a list call is not a good pattern
42
+ since the creds of metaflow default datastore might not be the same as the datastore the user wants to access.
43
+
44
+ There is a function that can help verify if the datastore set in the task metadata
45
+ is the same as the default datastore. If it is not, then we should shout warning messages
46
+ to the user. In case the permissions are the same, nothing wrong happens, if they are not then
47
+ user will have some hint in the logs to help them figure out the issue.
48
+ """
49
+ ...
50
+
51
+ class FlowNotRunningException(metaflow.exception.MetaflowException, metaclass=type):
52
+ ...
53
+
54
+ def storage_backend_from_flow(flow: "metaflow.FlowSpec"):
27
55
  ...
28
56
 
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.948626 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.459044 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,13 +1,13 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.894076 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.391435 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
8
8
 
9
- import typing
10
9
  import metaflow
10
+ import typing
11
11
  if typing.TYPE_CHECKING:
12
12
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
13
13
 
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.916954 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.415291 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -1,6 +1,6 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.919341 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.417687 #
5
5
  ######################################################################################################
6
6
 
@@ -1,7 +1,7 @@
1
1
  ######################################################################################################
2
2
  # Auto-generated Metaflow stub file #
3
- # MF version: 2.15.7.2+obcheckpoint(0.1.9);ob(v1) #
4
- # Generated on 2025-04-10T22:13:14.919388 #
3
+ # MF version: 2.15.7.2+obcheckpoint(0.2.0);ob(v1) #
4
+ # Generated on 2025-04-14T19:55:43.417772 #
5
5
  ######################################################################################################
6
6
 
7
7
  from __future__ import annotations
@@ -9,8 +9,8 @@ from __future__ import annotations
9
9
  import metaflow
10
10
  import typing
11
11
  if typing.TYPE_CHECKING:
12
- import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
13
12
  import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator
13
+ import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
14
14
 
15
15
  from ..checkpoints.decorator import CheckpointDecorator as CheckpointDecorator
16
16
  from ..checkpoints.decorator import CurrentCheckpointer as CurrentCheckpointer