ob-metaflow-stubs 6.0.3.117__py2.py3-none-any.whl → 6.0.3.118__py2.py3-none-any.whl

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/exceptions.pyi

@@ -0,0 +1,40 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.870144                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+import metaflow
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.exception
+
+from .....exception import MetaflowException as MetaflowException
+
+class TODOException(metaflow.exception.MetaflowException, metaclass=type):
+    def __init__(self, message):
+        ...
+    ...
+
+class KeyNotFoundError(metaflow.exception.MetaflowException, metaclass=type):
+    def __init__(self, url):
+        ...
+    ...
+
+class KeyNotCompatibleException(metaflow.exception.MetaflowException, metaclass=type):
+    def __init__(self, key, supported_types):
+        ...
+    ...
+
+class KeyNotCompatibleWithObjectException(metaflow.exception.MetaflowException, metaclass=type):
+    def __init__(self, key, store, message = None):
+        ...
+    ...
+
+class IncompatibleObjectTypeException(metaflow.exception.MetaflowException, metaclass=type):
+    def __init__(self, message):
+        ...
+    ...
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.910680                                                            #
+######################################################################################################
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/flowspec_utils.pyi

@@ -0,0 +1,71 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.910732                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow
+
+from ......metaflow_current import current as current
+from .identity_utils import TaskIdentifier as TaskIdentifier
+from .identity_utils import FlowNotRunningException as FlowNotRunningException
+
+TYPE_CHECKING: bool
+
+class ResolvedTask(tuple, metaclass=type):
+    """
+    ResolvedTask(flow_name, run_id, step_name, task_id, is_foreach, current_attempt, pathspec, control_task_pathspec)
+    """
+    @staticmethod
+    def __new__(_cls, flow_name, run_id, step_name, task_id, is_foreach, current_attempt, pathspec, control_task_pathspec):
+        """
+        Create new instance of ResolvedTask(flow_name, run_id, step_name, task_id, is_foreach, current_attempt, pathspec, control_task_pathspec)
+        """
+        ...
+    def __repr__(self):
+        """
+        Return a nicely formatted representation string
+        """
+        ...
+    def __getnewargs__(self):
+        """
+        Return self as a plain tuple.  Used by copy and pickle.
+        """
+        ...
+    ...
+
+class OriginInfo(tuple, metaclass=type):
+    """
+    OriginInfo(origin_run_id, origin_task_id, origin_attempt, is_foreach)
+    """
+    @staticmethod
+    def __new__(_cls, origin_run_id, origin_task_id, origin_attempt, is_foreach):
+        """
+        Create new instance of OriginInfo(origin_run_id, origin_task_id, origin_attempt, is_foreach)
+        """
+        ...
+    def __repr__(self):
+        """
+        Return a nicely formatted representation string
+        """
+        ...
+    def __getnewargs__(self):
+        """
+        Return self as a plain tuple.  Used by copy and pickle.
+        """
+        ...
+    ...
+
+def resolve_pathspec_for_flowspec(run: "metaflow.FlowSpec" = None):
+    ...
+
+def resolve_storage_backend(run: "metaflow.FlowSpec" = None):
+    ...
+
+def resolve_task_identifier(run: "metaflow.FlowSpec", gang_scheduled_task = False, gang_schedule_task_idf_index = 0):
+    ...
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/general.pyi

@@ -0,0 +1,56 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.912533                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.mf_extensions.obcheckpoint.plugins.machine_learning_utilities.utils.general
+
+
+class PathSize(tuple, metaclass=type):
+    """
+    PathSize(path, calculated_size, early_stopping, not_found)
+    """
+    @staticmethod
+    def __new__(_cls, path, calculated_size, early_stopping, not_found):
+        """
+        Create new instance of PathSize(path, calculated_size, early_stopping, not_found)
+        """
+        ...
+    def __repr__(self):
+        """
+        Return a nicely formatted representation string
+        """
+        ...
+    def __getnewargs__(self):
+        """
+        Return self as a plain tuple.  Used by copy and pickle.
+        """
+        ...
+    ...
+
+def replace_start_and_end_slash(string):
+    ...
+
+def get_path_size(start_path, early_stopping_limit = None) -> PathSize:
+    ...
+
+def warning_message(message, logger = None, ts = False, prefix = '[no-prefix]'):
+    ...
+
+def unit_convert(number, base_unit, convert_unit):
+    ...
+
+def is_json_serializable(value):
+    """
+    Check if value is JSON serializable.
+    """
+    ...
+
+def safe_serialize(data):
+    ...
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/identity_utils.pyi

@@ -0,0 +1,67 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.922066                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+import metaflow
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.exception
+    import metaflow
+
+from ......exception import MetaflowException as MetaflowException
+
+TYPE_CHECKING: bool
+
+MAX_HASH_LEN: int
+
+class AttemptEnum(object, metaclass=type):
+    @classmethod
+    def values(cls):
+        ...
+    ...
+
+class IdentityException(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+class UnhashableValueException(metaflow.exception.MetaflowException, metaclass=type):
+    def __init__(self, type):
+        ...
+    ...
+
+class FlowNotRunningException(metaflow.exception.MetaflowException, metaclass=type):
+    ...
+
+def pathspec_hash(pathspec: str):
+    ...
+
+def safe_hash(value):
+    """
+    Safely hash arbitrary values and a created a sha for that object, including those that are not natively hashable.
+    """
+    ...
+
+class TaskIdentifier(object, metaclass=type):
+    """
+    Any change to this class's core logic can create SEVERE backwards compatibility issues
+    since this class helps derive the task identifier for the checkpoints.
+    
+    IDEALLY, the identifier construction logic of this file should be kept as is.
+    """
+    @classmethod
+    def for_parallel_task_index(cls, run: "metaflow.FlowSpec", index: int):
+        """
+        This class is meant to mint a task-identifier for a parallel task based on the
+        index of the task in the gang.
+        """
+        ...
+    @classmethod
+    def from_flowspec(cls, run: "metaflow.FlowSpec"):
+        ...
+    def from_task(cls, task: "metaflow.Task", use_origin = True):
+        ...
+    ...
+

metaflow-stubs/mf_extensions/obcheckpoint/plugins/machine_learning_utilities/utils/tar_utils.pyi

@@ -0,0 +1,49 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.912150                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+import metaflow
+import typing
+if typing.TYPE_CHECKING:
+    import metaflow.exception
+
+from ......exception import MetaflowException as MetaflowException
+
+class TarballCreationError(metaflow.exception.MetaflowException, metaclass=type):
+    def __init__(self, message, error):
+        ...
+    ...
+
+def warning_message(message, logger = None, ts = False, prefix = '[@checkpoint][tarball-creator]'):
+    ...
+
+def os_error_safe_file_adding(tar, file_path, arcname):
+    ...
+
+def create_tarball_in_memory(source_paths: typing.Union[str, typing.List[str]], compression_method = 'gz', strict = False):
+    ...
+
+def create_tarball_on_disk(source_path: str, output_filename = None, compression_method = 'gz', strict = False):
+    """
+    Create a tarball of the specified file or directory.
+    
+    Parameters:
+    - source_path: The path to the file or directory to add to tarball.
+    - output_filename: The path where the tarball should be saved.
+    """
+    ...
+
+def extract_tarball(tarball_path_or_bytes, target_directory, compression_method = 'gz'):
+    """
+    Extract a tarball to the specified directory.
+    
+    Parameters:
+    - tarball_path_or_bytes: The path to the tarball to be extracted or the bytes of the tarball.
+    - target_directory: The directory where the contents of the tarball should be extracted.
+    """
+    ...
+

metaflow-stubs/mf_extensions/outerbounds/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.837750                                                            #
+######################################################################################################
+

metaflow-stubs/mf_extensions/outerbounds/profilers/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.850093                                                            #
+######################################################################################################
+

metaflow-stubs/mf_extensions/outerbounds/profilers/gpu.pyi

@@ -0,0 +1,130 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.850145                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+import typing
+
+from ....plugins.cards.card_modules.components import Markdown as Markdown
+from ....plugins.cards.card_modules.components import Table as Table
+from ....plugins.cards.card_modules.components import VegaChart as VegaChart
+from ....metaflow_current import current as current
+from ....plugins.cards.card_modules.components import Image as Image
+
+MEM_COLOR: str
+
+GPU_COLOR: str
+
+NVIDIA_TS_FORMAT: str
+
+MONITOR_FIELDS: list
+
+MONITOR: str
+
+class ProcessUUID(tuple, metaclass=type):
+    """
+    ProcessUUID(uuid, start_time, end_time)
+    """
+    @staticmethod
+    def __new__(_cls, uuid, start_time, end_time):
+        """
+        Create new instance of ProcessUUID(uuid, start_time, end_time)
+        """
+        ...
+    def __repr__(self):
+        """
+        Return a nicely formatted representation string
+        """
+        ...
+    def __getnewargs__(self):
+        """
+        Return self as a plain tuple.  Used by copy and pickle.
+        """
+        ...
+    ...
+
+class AsyncProcessManager(object, metaclass=type):
+    """
+    This class is responsible for managing the nvidia SMI subprocesses
+    """
+    @classmethod
+    def get(cls, procid):
+        ...
+    @classmethod
+    def spawn(cls, procid, cmd, file):
+        ...
+    @classmethod
+    def remove(cls, procid, delete_item = True):
+        ...
+    @classmethod
+    def cleanup(cls):
+        ...
+    @classmethod
+    def is_running(cls, procid):
+        ...
+    def __init__(self, processes: typing.Dict[str, typing.Dict]):
+        ...
+    ...
+
+class GPUMonitor(object, metaclass=type):
+    """
+    The `GPUMonitor` class is designed to monitor GPU usage.
+    
+    When an instance of `GPUMonitor` is created, it initializes with a specified `interval` and `duration`.
+    The `duration` is the timeperiod it will run the NVIDIA SMI command for and the `interval` is the timeperiod between each reading.
+    The class exposes a `_monitor_update_thread` method which runs as a background thread that continuously updates the GPU usage readings.
+    It will keep running unitl the `_finished` flag is set to `True`.
+    
+    The class will statefully manage the the spawned NVIDI-SMI processes.
+    It will start a new NVIDI-SMI process after the current one has ran for the specified `duration`.
+    At a time this class will only maintain readings for the `_current_process` and will have all the aggregated
+    readings for the past processes stored in the `_past_readings` dictionary.
+    When a process finishes completion, the readings are appended to the `_past_readings` dictionary and a new process is started.
+    
+    If the caller of this class wishes to read the GPU usage, they can call the `read` method which will return the readings in a dictionary format.
+    The `read` method will aggregate the readings from the `_current_readings` and `_past_readings`.
+    """
+    def __init__(self, interval = 1, duration = 300):
+        ...
+    @property
+    def _current_file(self):
+        ...
+    def get_file_name(self, uuid):
+        ...
+    def create_new_monitor(self):
+        ...
+    def clear_current_monitor(self):
+        ...
+    def current_process_has_ended(self):
+        ...
+    def current_process_is_running(self):
+        ...
+    def read(self):
+        ...
+    def cleanup(self):
+        ...
+    ...
+
+class GPUProfiler(object, metaclass=type):
+    def __init__(self, interval = 1, monitor_batch_duration = 200, artifact_name = 'gpu_profile_data', max_check_timeout = 60):
+        ...
+    def finish(self):
+        ...
+    ...
+
+class gpu_profile(object, metaclass=type):
+    def __init__(self, include_artifacts = True, artifact_prefix = 'gpu_profile_', interval = 1):
+        ...
+    def __call__(self, f):
+        ...
+    ...
+
+def translate_to_vegalite(tstamps, vals, description, y_label, legend, line_color = None, percentage_format = False):
+    ...
+
+def profile_plots(device_id, ts, gpu, mem_used, mem_total):
+    ...
+

metaflow-stubs/mf_extensions/outerbounds/remote_config.pyi

@@ -0,0 +1,42 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.905497                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+import typing
+if typing.TYPE_CHECKING:
+    import pathlib
+
+from ...exception import MetaflowException as MetaflowException
+from ...plugins.perimeters import get_perimeter_config_url_if_set_in_ob_config as get_perimeter_config_url_if_set_in_ob_config
+
+OBP_REMOTE_CONFIG_KEY: str
+
+HOSTNAME_KEY: str
+
+AUTH_KEY: str
+
+PERIMETER_KEY: str
+
+CONFIG_READ_ONCE_KEY: str
+
+def read_config_from_local() -> typing.Optional[pathlib.Path]:
+    ...
+
+def resolve_config_from_remote(remote_url: str, auth_token: str) -> typing.Dict[str, str]:
+    ...
+
+def init_config() -> typing.Dict[str, str]:
+    """
+    OSS Metaflow reads the config file on every step initialization. This is because OSS assumes config files change
+    relatively infrequently. We want to avoid config values changing between flow steps. Our solution to prevent this
+    is to read a config once and cache it on an environment variable. Environment variables carry over between steps
+    because steps are executed in subprocesses (local) or environments which expect environment variables to be set.
+    """
+    ...
+
+DEBUG_CONFIG: None
+

metaflow-stubs/mf_extensions/outerbounds/toplevel/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.837806                                                            #
+######################################################################################################
+

metaflow-stubs/mf_extensions/outerbounds/toplevel/global_aliases_for_metaflow_package.pyi

@@ -0,0 +1,17 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.837852                                                            #
+######################################################################################################
+
+from __future__ import annotations
+
+
+from .... import profilers as profilers
+
+def get_aws_client(module, with_error = False, role_arn = None, session_vars = None, client_params = None):
+    ...
+
+def S3(*args, **kwargs):
+    ...
+

metaflow-stubs/mflog/__init__.pyi

@@ -0,0 +1,6 @@
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.871761                                                            #
+######################################################################################################
+

metaflow-stubs/mflog/mflog.pyi

@@ -1,8 +1,8 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1)                               #
-# Generated on 2024-11-13T19:05:29.395124                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.871813                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
@@ -10,13 +10,60 @@ import typing
 if typing.TYPE_CHECKING:
     import datetime
 
+
 VERSION: bytes
 
 RE: bytes
 
+class MFLogline(tuple, metaclass=type):
+    """
+    MFLogline(should_persist, version, utc_tstamp_str, logsource, id, msg, utc_tstamp)
+    """
+    @staticmethod
+    def __new__(_cls, should_persist, version, utc_tstamp_str, logsource, id, msg, utc_tstamp):
+        """
+        Create new instance of MFLogline(should_persist, version, utc_tstamp_str, logsource, id, msg, utc_tstamp)
+        """
+        ...
+    def __repr__(self):
+        """
+        Return a nicely formatted representation string
+        """
+        ...
+    def __getnewargs__(self):
+        """
+        Return self as a plain tuple.  Used by copy and pickle.
+        """
+        ...
+    ...
+
 ISOFORMAT: str
 
 MISSING_TIMESTAMP: datetime.datetime
 
 MISSING_TIMESTAMP_STR: str
 
+def utc_to_local(x):
+    ...
+
+def decorate(source, line, version = b'0', now = None, lineid = None):
+    ...
+
+def is_structured(line):
+    ...
+
+def parse(line):
+    ...
+
+def set_should_persist(line):
+    ...
+
+def unset_should_persist(line):
+    ...
+
+def refine(line, prefix = None, suffix = None):
+    ...
+
+def merge_logs(logs):
+    ...
+

metaflow-stubs/multicore_utils.pyi

@@ -1,8 +1,8 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1)                               #
-# Generated on 2024-11-13T19:05:29.344336                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.832706                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
@@ -10,6 +10,7 @@ import typing
 if typing.TYPE_CHECKING:
     import typing
 
+
 class MulticoreException(Exception, metaclass=type):
     ...
 

metaflow-stubs/parameters.pyi

@@ -1,34 +1,22 @@
-##################################################################################
-#                       Auto-generated Metaflow stub file                        #
-# MF version: 2.12.27.1+obcheckpoint(0.1.2);ob(v1)                               #
-# Generated on 2024-11-13T19:05:29.345522                                        #
-##################################################################################
+######################################################################################################
+#                                 Auto-generated Metaflow stub file                                  #
+# MF version: 2.12.30.1+obcheckpoint(0.1.4);ob(v1)                                                   #
+# Generated on 2024-11-13T23:34:52.833933                                                            #
+######################################################################################################
 
 from __future__ import annotations
 
+import metaflow
 import typing
 if typing.TYPE_CHECKING:
-    import typing
     import metaflow.parameters
     import metaflow._vendor.click.types
-    import metaflow.exception
-
-class ParameterFieldFailed(metaflow.exception.MetaflowException, metaclass=type):
-    def __init__(self, name, field):
-        ...
-    ...
-
-class ParameterFieldTypeMismatch(metaflow.exception.MetaflowException, metaclass=type):
-    def __init__(self, msg):
-        ...
-    ...
+    import typing
 
-class MetaflowException(Exception, metaclass=type):
-    def __init__(self, msg = "", lineno = None):
-        ...
-    def __str__(self):
-        ...
-    ...
+from ._vendor import click as click
+from .exception import ParameterFieldFailed as ParameterFieldFailed
+from .exception import ParameterFieldTypeMismatch as ParameterFieldTypeMismatch
+from .exception import MetaflowException as MetaflowException
 
 class ParameterContext(tuple, metaclass=type):
     """
@@ -184,3 +172,5 @@ class Parameter(object, metaclass=type):
 def add_custom_parameters(deploy_mode = False):
     ...
 
+JSONType: JSONTypeClass
+