ob-metaflow-stubs 6.0.3.116__py2.py3-none-any.whl → 6.0.3.118__py2.py3-none-any.whl

metaflow-stubs/{metadata → metadata_provider}/metadata.pyi

@@ -1,31 +1,19 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.27.1+obcheckpoint(0.1.1);ob(v1)                               #
-# Generated on 2024-11-04T20:32:13.033616                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.851501                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
+import metaflow
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow.exception
-    import metaflow.metadata.metadata
+    import metaflow.metadata_provider.metadata
 
-class MetaflowInternalError(metaflow.exception.MetaflowException, metaclass=type):
-    ...
-
-class MetaflowTaggingError(metaflow.exception.MetaflowException, metaclass=type):
-    ...
-
-def validate_tag(tag):
-    """
-    - Tag must be either of bytes-type or unicode-type.
-    - If tag is of bytes-type, it must be UTF-8 decodable
-    - If tag is of unicode-type, it must be UTF-8 encodable
-    - Tag may not be empty string.
-    - Tag cannot be too long (500 chars)
-    """
-    ...
+from ..exception import MetaflowInternalError as MetaflowInternalError
+from ..exception import MetaflowTaggingError as MetaflowTaggingError
+from ..tagging_util import validate_tag as validate_tag
 
 class DataArtifact(tuple, metaclass=type):
     """

metaflow-stubs/metadata_provider/util.pyi

@@ -0,0 +1,19 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.851730                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+
+
+def copy_tree(src, dst, update = False):
+    ...
+
+def sync_local_metadata_to_datastore(metadata_local_dir, task_ds):
+    ...
+
+def sync_local_metadata_from_datastore(metadata_local_dir, task_ds):
+    ...
+

metaflow-stubs/metaflow_config.pyi

@@ -1,18 +1,13 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.27.1+obcheckpoint(0.1.1);ob(v1)                               #
-# Generated on 2024-11-04T20:32:12.975983                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.832289                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
 
-class MetaflowException(Exception, metaclass=type):
-    def __init__(self, msg = "", lineno = None):
-        ...
-    def __str__(self):
-        ...
-    ...
+from .exception import MetaflowException as MetaflowException
 
 DATASTORE_LOCAL_DIR: str
 
@@ -40,6 +35,8 @@ DEFAULT_SECRETS_BACKEND_TYPE: None
 
 DEFAULT_SECRETS_ROLE: None
 
+DEFAULT_FROM_DEPLOYMENT_IMPL: str
+
 USER: None
 
 DATASTORE_SYSROOT_LOCAL: None
@@ -240,8 +237,6 @@ AIRFLOW_KUBERNETES_KUBECONFIG_FILE: None
 
 AIRFLOW_KUBERNETES_KUBECONFIG_CONTEXT: None
 
-FROM_DEPLOYMENT_IMPL: str
-
 CONDA_PACKAGE_S3ROOT: None
 
 CONDA_PACKAGE_AZUREROOT: None

metaflow-stubs/metaflow_current.pyi

@@ -1,21 +1,22 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.27.1+obcheckpoint(0.1.1);ob(v1)                               #
-# Generated on 2024-11-04T20:32:13.082093                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.934305                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
 import typing
 if typing.TYPE_CHECKING:
-    import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator
+    import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator
+    import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core
+    import metaflow.plugins.cards.component_serializer
+    import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
+    import metaflow.events
     import metaflow
     import typing
     import metaflow.metaflow_current
-    import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core
-    import metaflow.events
-    import metaflow.plugins.cards.component_serializer
-    import metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator
+
 
 TYPE_CHECKING: bool
 
@@ -224,28 +225,108 @@ class Current(object, metaclass=type):
         """
         ...
     @property
-    def graph(_):
+    def graph(self):
         ...
     @property
-    def card(self) -> "metaflow.plugins.cards.component_serializer.CardComponentCollector":
+    def huggingface_hub(self) -> "metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator.HuggingfaceRegistry":
         """
-        (only in the presence of the @card decorator)
+        (only in the presence of the @huggingface_hub decorator)
         
-        The `@card` decorator makes the cards available through the `current.card`
-        object. If multiple `@card` decorators are present, you can add an `ID` to
-        distinguish between them using `@card(id=ID)` as the decorator. You will then
-        be able to access that specific card using `current.card[ID].
+        The `@huggingface_hub` injects a `huggingface_hub` object into the `current` object. This object provides syntactic sugar
+        over [huggingface_hub](https://github.com/huggingface/huggingface_hub)'s
+        [snapshot_download](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/file_download#huggingface_hub.snapshot_download) function.
+        The `current.huggingface_hub.snapshot_download` function downloads objects from huggingface hub and saves them to the Metaflow's datastore under the
+        `<repo_type>/<repo_id>` name. The `repo_type` is by default `model` and can be overriden by passing the `repo_type` parameter to the `snapshot_download` function.
         
-        Methods available are `append` and `extend`
+        
+        Usage:
+        ------
+        
+        **Usage: creating references of models from huggingface that may be loaded in downstream steps**
+        ```python
+            @huggingface_hub
+            @step
+            def pull_model_from_huggingface(self):
+                # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
+                # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
+                # with the same `repo_id` in the backend storage, it will not download the model again. The return
+                # value of the function is a reference to the model in the backend storage.
+                # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
+        
+                self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
+                self.llama_model = current.huggingface_hub.snapshot_download(
+                    repo_id=self.model_id,
+                    allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
+                )
+                self.next(self.train)
+        ```
+        
+        **Usage: loading models directly from huggingface hub or from cache (from metaflow's datastore)**
+        ```python
+            @huggingface_hub(load=["mistralai/Mistral-7B-Instruct-v0.1"])
+            @step
+            def pull_model_from_huggingface(self):
+                path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
+        ```
+        
+        ```python
+            @huggingface_hub(load=[("mistralai/Mistral-7B-Instruct-v0.1", "/my-directory"), ("myorg/mistral-lora, "/my-lora-directory")])
+            @step
+            def finetune_model(self):
+                path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
+                # path_to_model will be /my-directory
+        ```
+        
+        ```python
+            # Takes all the arguments passed to `snapshot_download`
+            # except for `local_dir`
+            @huggingface_hub(load=[
+                {
+                    "repo_id": "mistralai/Mistral-7B-Instruct-v0.1",
+                },
+                {
+                    "repo_id": "myorg/mistral-lora",
+                    "repo_type": "model",
+                },
+            ])
+            @step
+            def finetune_model(self):
+                path_to_model = current.huggingface_hub.loaded["mistralai/Mistral-7B-Instruct-v0.1"]
+                # path_to_model will be /my-directory
+        ```
+        """
+        ...
+    @property
+    def parallel(self) -> "metaflow.metaflow_current.Parallel":
+        """
+        (only in the presence of the @parallel decorator)
+        
+        Returns a namedtuple with relevant information about the parallel task.
         
         Returns
         -------
-        CardComponentCollector
-            The or one of the cards attached to this step.
+        Parallel
+            `namedtuple` with the following fields:
+                - main_ip (`str`)
+                    The IP address of the control task.
+                - num_nodes (`int`)
+                    The total number of tasks created by @parallel
+                - node_index (`int`)
+                    The index of the current task in all the @parallel tasks.
+                - control_task_id (`Optional[str]`)
+                    The task ID of the control task. Available to all tasks.
         """
         ...
     @property
-    def model(self) -> "metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core.ModelSerializer":
+    def is_parallel(self) -> bool:
+        """
+        (only in the presence of the @parallel decorator)
+        
+        True if the current step is a @parallel step.
+        """
+        ...
+    @property
+    def model(self) -> "metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.modeling_utils.core.ModelSerializer":
         """
         (only in the presence of the @model decorator)
         
@@ -306,69 +387,7 @@ class Current(object, metaclass=type):
         """
         ...
     @property
-    def parallel(self) -> "metaflow.metaflow_current.Parallel":
-        """
-        (only in the presence of the @parallel decorator)
-        
-        Returns a namedtuple with relevant information about the parallel task.
-        
-        Returns
-        -------
-        Parallel
-            `namedtuple` with the following fields:
-                - main_ip (`str`)
-                    The IP address of the control task.
-                - num_nodes (`int`)
-                    The total number of tasks created by @parallel
-                - node_index (`int`)
-                    The index of the current task in all the @parallel tasks.
-                - control_task_id (`Optional[str]`)
-                    The task ID of the control task. Available to all tasks.
-        """
-        ...
-    @property
-    def is_parallel(self) -> bool:
-        """
-        (only in the presence of the @parallel decorator)
-        
-        True if the current step is a @parallel step.
-        """
-        ...
-    @property
-    def huggingface_hub(self) -> "metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.hf_hub.decorator.HuggingfaceRegistry":
-        """
-        (only in the presence of the @huggingface_hub decorator)
-        
-        The `@huggingface_hub` injects a `huggingface_hub` object into the `current` object. This object provides syntactic sugar
-        over [huggingface_hub](https://github.com/huggingface/huggingface_hub)'s
-        [snapshot_download](https://huggingface.co/docs/huggingface_hub/main/en/package_reference/file_download#huggingface_hub.snapshot_download) function.
-        The `current.huggingface_hub.snapshot_download` function downloads objects from huggingface hub and saves them to the Metaflow's datastore under the
-        `<repo_type>/<repo_id>` name. The `repo_type` is by default `model` and can be overriden by passing the `repo_type` parameter to the `snapshot_download` function.
-        
-        
-        Usage:
-        ------
-        ```python
-            @huggingface_hub
-            @step
-            def pull_model_from_huggingface(self):
-                # `current.huggingface_hub.snapshot_download` downloads the model from the Hugging Face Hub
-                # and saves it in the backend storage based on the model's `repo_id`. If there exists a model
-                # with the same `repo_id` in the backend storage, it will not download the model again. The return
-                # value of the function is a reference to the model in the backend storage.
-                # This reference can be used to load the model in the subsequent steps via `@model(load=["llama_model"])`
-        
-                self.model_id = "mistralai/Mistral-7B-Instruct-v0.1"
-                self.llama_model = current.huggingface_hub.snapshot_download(
-                    repo_id=self.model_id,
-                    allow_patterns=["*.safetensors", "*.json", "tokenizer.*"],
-                )
-                self.next(self.train)
-        ```
-        """
-        ...
-    @property
-    def checkpoint(self) -> "metaflow_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator.CurrentCheckpointer":
+    def checkpoint(self) -> "metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.decorator.CurrentCheckpointer":
         """
         (only in the presence of the @checkpoint decorator)
         
@@ -429,16 +448,21 @@ class Current(object, metaclass=type):
         """
         ...
     @property
-    def trigger(self) -> "metaflow.events.Trigger":
+    def card(self) -> "metaflow.plugins.cards.component_serializer.CardComponentCollector":
         """
-        (only in the presence of the @trigger, or @trigger_on_finish decorators)
+        (only in the presence of the @card decorator)
         
-        Returns `Trigger` if the current run is triggered by an event
+        The `@card` decorator makes the cards available through the `current.card`
+        object. If multiple `@card` decorators are present, you can add an `ID` to
+        distinguish between them using `@card(id=ID)` as the decorator. You will then
+        be able to access that specific card using `current.card[ID].
+        
+        Methods available are `append` and `extend`
         
         Returns
         -------
-        Trigger
-            `Trigger` if triggered by an event
+        CardComponentCollector
+            The or one of the cards attached to this step.
         """
         ...
     @property
@@ -508,6 +532,19 @@ class Current(object, metaclass=type):
             True if the flow is deployed with `--production`.
         """
         ...
+    @property
+    def trigger(self) -> "metaflow.events.Trigger":
+        """
+        (only in the presence of the @trigger_on_finish, or @trigger decorators)
+        
+        Returns `Trigger` if the current run is triggered by an event
+        
+        Returns
+        -------
+        Trigger
+            `Trigger` if triggered by an event
+        """
+        ...
     ...
 
 current: Current

metaflow-stubs/mf_extensions/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.837688                                                            #
+######################################################################################################
+

metaflow-stubs/mf_extensions/obcheckpoint/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.846039                                                            #
+######################################################################################################
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.846103                                                            #
+######################################################################################################
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.846162                                                            #
+######################################################################################################
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.846219                                                            #
+######################################################################################################
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/checkpoint_storage.pyi

@@ -0,0 +1,129 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.911726                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+import metaflow
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
+    import metaflow.datastore.datastore_storage
+    import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core
+    import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.checkpoints.checkpoint_storage
+
+from ..exceptions import KeyNotCompatibleWithObjectException as KeyNotCompatibleWithObjectException
+from ..utils.identity_utils import pathspec_hash as pathspec_hash
+from ..utils.general import replace_start_and_end_slash as replace_start_and_end_slash
+from ..datastore.core import allow_safe as allow_safe
+from ..datastore.core import DatastoreInterface as DatastoreInterface
+from ..datastore.core import ObjectStorage as ObjectStorage
+from ..datastore.core import STORAGE_FORMATS as STORAGE_FORMATS
+from ..datastore.exceptions import DatastoreReadInitException as DatastoreReadInitException
+from ..datastore.exceptions import DatastoreWriteInitException as DatastoreWriteInitException
+from ..datastore.exceptions import DatastoreNotReadyException as DatastoreNotReadyException
+from ..datastructures import CheckpointArtifact as CheckpointArtifact
+from ..datastore.utils import safe_serialize as safe_serialize
+
+DEFAULT_NAME: str
+
+CHECKPOINTS_STORAGE_PREFIX: str
+
+DEFAULT_STORAGE_FORMAT: str
+
+ARTIFACT_STORE_NAME: str
+
+METADATA_STORE_NAME: str
+
+ARTIFACT_METADATA_STORE_NAME: str
+
+class CheckpointsPathComponents(tuple, metaclass=type):
+    """
+    CheckpointsPathComponents(flow_name, step_name, scope, task_identifier, pathspec_hash, attempt, name, version_id, is_metadata, key_name, root_prefix)
+    """
+    @staticmethod
+    def __new__(_cls, flow_name, step_name, scope, task_identifier, pathspec_hash, attempt, name, version_id, is_metadata, key_name, root_prefix):
+        """
+        Create new instance of CheckpointsPathComponents(flow_name, step_name, scope, task_identifier, pathspec_hash, attempt, name, version_id, is_metadata, key_name, root_prefix)
+        """
+        ...
+    def __repr__(self):
+        """
+        Return a nicely formatted representation string
+        """
+        ...
+    def __getnewargs__(self):
+        """
+        Return self as a plain tuple.  Used by copy and pickle.
+        """
+        ...
+    ...
+
+def decompose_key_artifact_metadata_store(key) -> CheckpointsPathComponents:
+    ...
+
+def decompose_key_artifact_store(key) -> CheckpointsPathComponents:
+    """
+    Convert Key into Path Components.
+    PATH COMPONENTS: mf.checkpoints/artifacts/<flow_name>/<step_name>/<scope>/<task_identifier>/<pathspec_hash>.<attempt>.<name>.<version_id>
+    """
+    ...
+
+def decompose_key_metadata_store(key) -> CheckpointsPathComponents:
+    """
+    Convert Key into Path Components.
+    PATH COMPONENTS: mf.checkpoints/artifacts/<flow_name>/<step_name>/<scope>/<task_identifier>/<pathspec_hash>.<attempt>.<name>.<version_id>
+    """
+    ...
+
+class CheckpointDatastore(metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.DatastoreInterface, metaclass=type):
+    """
+    Consisits of 3 main components:
+    - `artifact_store`: This is where the checkpoint artifacts are stored.
+        - This key to the checkpoint in this store becomes the "key for the checkpoint"
+    - `metadata_store`: This is where the metadata of the checkpoint is stored it based on the currently executing task (path structure resembles that of a metaflow pathspec).
+        - This store helps retrieve information about all the checkpoints stored for a task during the execution.
+    - `artifact_metadatastore`: This is similar to the metadata store but holds a pathstructure similar to the artifact store.
+        - this store helps reverse lookup the Checkpoint metadata object from the checkpoint key.
+    """
+    @property
+    def metadata_ready(self):
+        ...
+    @property
+    def artifact_ready(self):
+        ...
+    def set_root_prefix(self, root_prefix):
+        ...
+    @classmethod
+    def init_read_store(cls, storage_backend: metaflow.datastore.datastore_storage.DataStoreStorage, pathspec = None, checkpoint_key = None):
+        """
+        This will initialize the datastore for reading.
+        
+        - If there is only the pathspec that's provided then it can mean the user is doing a list operations
+        - if only the checkpoint_key is provided then it can mean the user is trying to load a specific checkpoint
+        """
+        ...
+    def create_key_name(self, *args):
+        ...
+    @classmethod
+    def init_write_store(cls, storage_backend: metaflow.datastore.datastore_storage.DataStoreStorage, pathspec, scope, task_identifier):
+        ...
+    def save(self, local_path: str, attempt, version_id, name = 'mfchckpt', metadata = {}, set_latest = True, storage_format = 'files') -> metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact:
+        ...
+    def latest(*args, **kwargs):
+        ...
+    def load(self, local_path, version_id, attempt, name, storage_format = 'files'):
+        ...
+    def load_metadata(self, attempt, version_id, name = 'mfchckpt') -> dict:
+        ...
+    def list(self, name = None, attempt = None, within_task = True):
+        ...
+    @classmethod
+    def decompose_key(cls, key) -> CheckpointsPathComponents:
+        ...
+    def __init__(self, artifact_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, metadata_store: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage, artifact_metadatastore: metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastore.core.ObjectStorage):
+        ...
+    ...
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/checkpoints/constructors.pyi

@@ -0,0 +1,26 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.869613                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures
+
+from .core import Checkpointer as Checkpointer
+from .core import WriteResolver as WriteResolver
+from .core import ReadResolver as ReadResolver
+from ..datastructures import CheckpointArtifact as CheckpointArtifact
+
+TYPE_CHECKING: bool
+
+CHECKPOINT_UID_ENV_VAR_NAME: str
+
+DEFAULT_NAME: str
+
+def load_checkpoint(checkpoint: typing.Union[metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.datastructures.CheckpointArtifact, dict, str], local_path: str):
+    ...
+