zarth-utils 1.0__tar.gz

zarth_utils-1.0/PKG-INFO

@@ -0,0 +1,32 @@
+Metadata-Version: 2.1
+Name: zarth_utils
+Version: 1.0
+Summary: Package used for my personal development on ML projects.
+Requires-Python: >=3.7
+Description-Content-Type: text/x-rst
+Requires-Dist: matplotlib
+Requires-Dist: tqdm
+Requires-Dist: pandas
+Requires-Dist: numpy
+Requires-Dist: joblib
+Requires-Dist: gitpython
+Requires-Dist: ipython
+Requires-Dist: wandb
+Requires-Dist: scikit-learn
+Requires-Dist: gitpython
+Requires-Dist: ipython
+Provides-Extra: all
+Requires-Dist: nltk; extra == "all"
+Requires-Dist: torch; extra == "all"
+Requires-Dist: tensorflow; extra == "all"
+Requires-Dist: jupyterlab; extra == "all"
+Provides-Extra: hf
+Requires-Dist: transformers; extra == "hf"
+Requires-Dist: accelerate; extra == "hf"
+Requires-Dist: trl; extra == "hf"
+Requires-Dist: datasets; extra == "hf"
+Requires-Dist: diffusers; extra == "hf"
+Requires-Dist: tokenizers; extra == "hf"
+Requires-Dist: huggingface_hub; extra == "hf"
+
+Package used for my personal development on ML projects.

zarth_utils-1.0/README.md

@@ -0,0 +1 @@
+Package used for my personal development on ML projects.

zarth_utils-1.0/pyproject.toml

@@ -0,0 +1,21 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "zarth_utils"
+description = "Package used for my personal development on ML projects."
+version = "1.0"
+dynamic = ["readme", "dependencies"]
+requires-python = ">=3.7"
+
+[project.optional-dependencies]
+all = ["nltk", "torch", "tensorflow", "jupyterlab"]
+hf = ["transformers", "accelerate", "trl", "datasets", "diffusers", "tokenizers", "huggingface_hub"]
+
+[tool.setuptools.packages.find]
+include = ["zarth_utils"]
+
+[tool.setuptools.dynamic]
+readme = { file = "README.md" }
+dependencies = { file = "requirements.txt" }

zarth_utils-1.0/requirements.txt

@@ -0,0 +1,11 @@
+matplotlib
+tqdm
+pandas
+numpy
+joblib
+gitpython
+ipython
+wandb
+scikit-learn
+gitpython
+ipython

zarth_utils-1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

zarth_utils-1.0/tests/test_config.py

@@ -0,0 +1,37 @@
+from unittest import TestCase
+from zarth_utils.config import Config
+
+
+class TestConfig(TestCase):
+    def test_config(self):
+        config = Config(default_config_dict={
+            "a": "a",
+            "b": {
+                "c": "c",
+                "d": "d",
+                "e": {
+                    "f": "f"
+                }
+            }
+        })
+        config.a = "A"
+        config["b.c"] = "X"
+        config.b.c = "Y"
+        config.b["c"] = "C"
+        config["b"]["d"] = "D"
+        config["b"].e["f"] = "F"
+        self.assertEqual(config.a, "A")
+        self.assertEqual(config.b.c, "C")
+        self.assertEqual(config.b.d, "D")
+        self.assertEqual(config.b.e.f, "F")
+        self.assertEqual(config["a"], "A")
+        self.assertEqual(config["b"]["c"], "C")
+        self.assertEqual(config["b"]["d"], "D")
+        self.assertEqual(config["b"]["e"]["f"], "F")
+        self.assertEqual(config.get("a"), "A")
+        self.assertEqual(config.get("b.c"), "C")
+        self.assertEqual(config.get("b.d"), "D")
+        self.assertEqual(config.get("b.e.f"), "F")
+        self.assertEqual(config["b"].e["f"], "F")
+        self.assertEqual(config.b.e["f"], "F")
+        self.assertEqual(config.b["e"].f, "F")

zarth_utils-1.0/zarth_utils/__init__.py

@@ -0,0 +1,3 @@
+from importlib import metadata
+
+__version__ = metadata.version("zarth_utils")

zarth_utils-1.0/zarth_utils/config.py

@@ -0,0 +1,268 @@
+import os
+import json
+import argparse
+import logging
+
+import yaml
+
+from .general_utils import get_random_time_stamp, makedir_if_not_exist
+from .logger import logging_info
+
+try:
+    import wandb
+except ModuleNotFoundError as err:
+    logging.warning("WandB not installed!")
+
+dir_configs = os.path.join(os.getcwd(), "configs")
+
+
+def smart_load(path_file):
+    if path_file.endswith("json"):
+        return json.load(open(path_file, "r", encoding="utf-8"))
+    elif path_file.endswith("yaml") or path_file.endswith("yml"):
+        return yaml.safe_load(open(path_file, "r", encoding="utf-8"))
+    else:
+        logging.warning("Un-identified file type. It will be processed as json by default.")
+        return json.load(open(path_file, "r", encoding="utf-8"))
+
+
+class NestedDict(dict):
+    def __init__(self, *args, **kwargs):
+        """
+        Every element could be visited by either attribute or dict manner.
+
+        Examples:
+            >>> a = NestedDict()
+            >>> a["b"]["c"] = 1
+            >>> a.b.c
+            1
+            >>> a.b.d = 2
+            >>> a["b"]["d"]
+            2
+        """
+        super(NestedDict, self).__init__(*args, **kwargs)
+        for k in dict.keys(self):
+            if type(k) is dict:
+                dict.__setitem__(self, k, NestedDict(k))
+
+    def __getattr__(self, item):
+        return self[item]
+
+    def __setattr__(self, key, value):
+        self[key] = value
+
+    def __getitem__(self, key):
+        ret = self
+        for k in key.split("."):
+            ret = dict.__getitem__(ret, k)
+        return ret
+
+    def __setitem__(self, key, value):
+        key_list = key.split(".")
+        cur = self
+        for i in range(len(key_list)):
+            key = key_list[i]
+            if i == len(key_list) - 1:
+                dict.__setitem__(cur, key, value)
+            else:
+                if key in dict.keys(cur):
+                    assert type(dict.__getitem__(cur, key)) is NestedDict
+                else:
+                    dict.__setitem__(cur, key, NestedDict())
+                cur = dict.__getitem__(cur, key)
+
+    def update(self, new_dict, prefix=None):
+        for k in new_dict:
+            key = ".".join([prefix, k]) if prefix is not None else k
+            value = new_dict[k]
+            if type(value) is dict or type(value) is NestedDict:
+                self.update(value, prefix=key)
+            else:
+                self[key] = value
+
+    def keys(self, cur=None, prefix=None):
+        if cur is None:
+            cur = self
+
+        ret = []
+        for k in dict.keys(cur):
+            v = cur[k]
+            new_prefix = ".".join([prefix, k]) if prefix is not None else k
+            if type(v) is dict or type(v) is NestedDict:
+                ret += self.keys(cur=v, prefix=new_prefix)
+            else:
+                ret.append(new_prefix)
+        return ret
+
+    def get(self, item, default_value=None):
+        if item in self.keys():
+            return self[item]
+        return default_value
+
+    def show(self):
+        """
+        Show all the configs in logging. If get_logger is used before, then the outputs will also be in the log file.
+        """
+        logging_info("\n%s" % json.dumps(self._nested_dict, sort_keys=True, indent=4, separators=(',', ': ')))
+
+    def to_dict(self):
+        """
+        Return the config as a dict
+        :return: config dict
+        :rtype: dict
+        """
+        return self._nested_dict
+
+    def dump(self, path_dump=None):
+        """
+        Dump the config in the path_dump.
+        :param path_dump: the path to dump the config
+        :type path_dump: str
+        """
+        if path_dump is None:
+            makedir_if_not_exist(dir_configs)
+            path_dump = os.path.join(dir_configs, "%s.json" % get_random_time_stamp())
+        path_dump = "%s.json" % path_dump if not path_dump.endswith(".json") else path_dump
+        assert not os.path.exists(path_dump)
+        with open(path_dump, "w", encoding="utf-8") as fout:
+            json.dump(self._nested_dict, fout)
+
+
+class Config(NestedDict):
+    def __init__(self, default_config_file=None, default_config_dict=None, use_argparse=True, use_wandb=False):
+        """
+        Initialize the config. Note that either default_config_dict or default_config_file in json format must be
+        provided! The keys will be transferred to argument names, and the type will be automatically detected. The
+        priority is ``the user specified parameter (if the use_argparse is True)'' > ``user specified config file (if
+        the use_argparse is True)'' > ``default config dict'' > ``default config file''.
+
+        Examples:
+        default_config_dict = {"lr": 0.01, "optimizer": "sgd", "num_epoch": 30, "use_early_stop": False}
+        Then the following corresponding arguments will be added in this function if use_argparse is True:
+        parser.add_argument("--lr", type=float)
+        parser.add_argument("--optimizer", type=str)
+        parser.add_argument("--num_epoch", type=int)
+        parser.add_argument("--use_early_stop", action="store_true", default=False)
+        parser.add_argument("--no-use_early_stop", dest="use_early_stop", action="store_false")
+
+        :param default_config_dict: the default config dict
+        :type default_config_dict: dict
+        :param default_config_file: the default config file path
+        :type default_config_file: str
+        :param use_argparse: whether use argparse to parse the config
+        :type use_argparse: bool
+        :param use_wandb: whether init wandb with parent directory as project name and exp_name as run name
+        :type use_wandb: bool
+        """
+        super(Config, self).__init__()
+
+        # load from default config file
+        if default_config_dict is None and default_config_file is None:
+            if os.path.exists(os.path.join(os.getcwd(), "default_config.json")):
+                default_config_file = os.path.join(os.getcwd(), "default_config.json")
+            else:
+                logging.error("Either default_config_file or default_config_dict must be provided!")
+                raise NotImplementedError
+
+        if default_config_file is not None:
+            self.update(smart_load(default_config_file))
+        if default_config_dict is not None:
+            self.update(default_config_dict)
+
+        # transform the param terms into argparse
+        if use_argparse:
+            parser = argparse.ArgumentParser()
+            parser.add_argument("--config_file", type=str, default=None)
+            # add argument parser
+            for name_param in self.keys():
+                value_param = self[name_param]
+                if type(value_param) is bool:
+                    parser.add_argument("--%s" % name_param, action="store_true", default=None)
+                    parser.add_argument("--no-%s" % name_param, dest="%s" % name_param,
+                                        action="store_false", default=None)
+                elif type(value_param) is list:
+                    parser.add_argument("--%s" % name_param, type=type(value_param[0]), nargs="+", default=None)
+                else:
+                    parser.add_argument("--%s" % name_param, type=type(value_param), default=None)
+            args = parser.parse_args()
+
+            updated_parameters = dict()
+            args_dict = vars(args)
+            for k in vars(args):
+                if k != "config_file" and args_dict[k] is not None:
+                    updated_parameters[k] = args_dict[k]
+
+            if args.config_file is not None:
+                self.update(smart_load(args.config_file))
+
+            self.update(updated_parameters)
+
+        if use_wandb:
+            wandb.login()
+            wandb.init(
+                project=os.path.split(os.getcwd())[-1],
+                name=self["exp_name"],
+                config=self.to_dict()
+            )
+
+
+def get_parser():
+    """
+    Get a simple parser for argparse, which already contains the config_file argument.
+    """
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config_file", type=str, default=None)
+    return parser
+
+
+def parser2config(parser):
+    """
+    Parse the arguments from parser into a config.
+    """
+    return args2config(parser.parse_args())
+
+
+def args2config(args):
+    """
+    Parse the arguments from args into a config.
+    """
+    args_dict = vars(args)
+    return Config(default_config_dict=args_dict, use_argparse=False)
+
+
+def is_configs_same(config_a, config_b, ignored_keys=("load_epoch",)):
+    """
+    Judge whether two configs are the same.
+    Args:
+        config_a: the first config
+        config_b: the second config
+        ignored_keys: thes keys that will be ignored when comparing
+
+    Returns: True if the two configs are the same, False otherwise.
+
+    """
+    config_a, config_b = config_a.to_dict(), config_b.to_dict()
+
+    # make sure config A is always equal or longer than config B
+    if len(config_a.keys()) < len(config_b.keys()):
+        swap_var = config_a
+        config_a = config_b
+        config_b = swap_var
+
+    if len(config_a.keys() - config_b.keys()) > 1:
+        logging.error(
+            "Different config numbers: %d (Existing) : %d (New)!" % (len(config_a.keys()), len(config_b.keys())))
+        return False
+    elif len(config_a.keys() - config_b.keys()) == 1 and (config_a.keys() - config_b.keys())[0] != "config_file":
+        logging.error(
+            "Different config numbers: %d (Existing) : %d (New)!" % (len(config_a.keys()), len(config_b.keys())))
+        return False
+    else:
+        for i in config_a.keys() & config_b.keys():
+            _ai = tuple(config_a[i]) if type(config_a[i]) == list else config_a[i]
+            _bi = tuple(config_b[i]) if type(config_b[i]) == list else config_b[i]
+            if _ai != _bi and i not in ignored_keys:
+                logging.error("Mismatch in %s: %s (Existing) - %s (New)" % (str(i), str(config_a[i]), str(config_b[i])))
+                return False
+
+    return True

zarth_utils-1.0/zarth_utils/drawer.py

@@ -0,0 +1,192 @@
+import os
+import numpy as np
+
+from matplotlib.pyplot import figure
+
+from .general_utils import get_random_time_stamp, makedir_if_not_exist
+
+dir_figures = os.path.join(os.getcwd(), "figures")
+makedir_if_not_exist(dir_figures)
+
+
+class Drawer:
+    def __init__(self, num_row=1, num_col=1, unit_length=10, unit_row_length=None, unit_col_length=None):
+        """
+        Init the drawer with the (width=num_col*unit_length, height=num_row*unit_length).
+        :param num_row: the number of rows
+        :type num_row: int
+        :param num_col: the number of columns
+        :type num_col: int
+        :param unit_length: the length of unit
+        :type unit_length: float
+        :param unit_row_length: the length of unit for rows
+        :param unit_col_length: the length of unit for cols
+        """
+        self.num_row = num_row
+        self.num_col = num_col
+        unit_row_length = unit_length if unit_row_length is None else unit_row_length
+        unit_col_length = unit_length if unit_col_length is None else unit_col_length
+        self.figure = figure(figsize=(num_col * unit_row_length, num_row * unit_col_length))
+
+    def add_one_empty_axes(self, index=1, nrows=None, ncols=None,
+                           title="", xlabel="", ylabel="", fontsize=15, xlim=None, ylim=None):
+        """
+        Draw one axes, which can be understood as a sub-figure.
+        :param index: The subplot will take the index position on a grid with nrows rows and ncols columns.
+        :type index: int
+        :param nrows: the number of rows in the figure
+        :type nrows: int
+        :param ncols: the number of columns in the figure
+        :type ncols: int
+        :param title: the title of the axes
+        :type title: str
+        :param xlabel: the label for x axis
+        :type xlabel: str
+        :param ylabel: the label for x axis
+        :type ylabel: str
+        :param fontsize: the size of the fonts
+        :param xlim: the range of x axis, (low, upp)
+        :param ylim: the range of y axis, (low, upp)
+        :return:
+        :rtype:
+        """
+        nrows = self.num_row if nrows is None else nrows
+        ncols = self.num_col if ncols is None else ncols
+
+        ax = self.figure.add_subplot(nrows, ncols, index)
+
+        ax.set_xlabel(xlabel, fontsize=fontsize)
+        ax.set_ylabel(ylabel, fontsize=fontsize)
+        ax.set_title(title, fontsize=fontsize)
+        ax.xaxis.set_tick_params(labelsize=fontsize)
+        ax.yaxis.set_tick_params(labelsize=fontsize)
+        if xlim is not None:
+            ax.set_xlim(*xlim)
+        if ylim is not None:
+            ax.set_ylim(*ylim)
+
+        return ax
+
+    def draw_one_axes(self, x, y, labels=None, *, index=1, nrows=None, ncols=None,
+                      title="", xlabel="", ylabel="", use_marker=False, linewidth=6,
+                      fontsize=15, xlim=None, ylim=None, smooth=0, mode="plot", **kwargs):
+        """
+        Draw one axes, which can be understood as a sub-figure.
+        :param x: the data for x axis, list
+        :param y: the data for y axis, list of line lists. e.g. [[1, 2, 3], [2, 3, 1]], list
+        :param labels: the list of labels of each line, list
+        :param index: The subplot will take the index position on a grid with nrows rows and ncols columns.
+        :type index: int
+        :param nrows: the number of rows in the figure
+        :type nrows: int
+        :param ncols: the number of columns in the figure
+        :type ncols: int
+        :param title: the title of the axes
+        :type title: str
+        :param xlabel: the label for x axis
+        :type xlabel: str
+        :param ylabel: the label for x axis
+        :type ylabel: str
+        :param use_marker: whether use markers to mark the points, default=False
+        :type use_marker: bool
+        :param linewidth: the width of the lines for mode "plot", or the size of the points for mode "scatter"
+        :param fontsize: the size of the fonts
+        :param xlim: the range of x axis, (low, upp)
+        :param ylim: the range of y axis, (low, upp)
+        :param smooth: smooth the line with neighbours
+        :param mode: "plot" or "scatter"
+        :return:
+        :rtype:
+        """
+        ax = self.add_one_empty_axes(index, nrows, ncols, title, xlabel, ylabel,
+                                     fontsize, xlim, ylim)
+
+        format_generator = self.get_format(use_marker)
+        for i, yi in enumerate(y):
+            if len(x) == len(y) and type(x[0]) is list:
+                xi = x[i]
+            elif len(x) == len(y[0]) and type(x[0]) is not list:
+                xi = x
+            else:
+                raise NotImplementedError
+
+            if smooth != 0:
+                yi_smoothed = []
+                for j, yij in enumerate(yi):
+                    _r = min(j + smooth, len(yi) - 1)
+                    _l = max(j - smooth, 0)
+                    yij = sum(yi[_l: _r]) / (_r - _l)
+                    yi_smoothed.append(yij)
+                yi = yi_smoothed
+
+            len_no_nan = 0
+            while len_no_nan < len(yi) and not (np.isnan(yi[len_no_nan]) or np.isinf(yi[len_no_nan])):
+                len_no_nan += 1
+            if len_no_nan == 0:
+                continue
+
+            fmt = next(format_generator)
+
+            if labels is not None:
+                kwargs["label"] = labels[i]
+            if mode == "plot":
+                kwargs["linewidth"] = linewidth
+
+            if mode == "plot":
+                ax.plot(xi[:len_no_nan], yi[:len_no_nan], fmt, **kwargs)
+            elif mode == "scatter":
+                ax.scatter(xi[:len_no_nan], yi[:len_no_nan], c=fmt[0], s=linewidth, **kwargs)
+            else:
+                raise NotImplementedError
+
+        if labels is not None:
+            ax.legend(fontsize=fontsize)
+
+        return ax
+
+    def show(self):
+        """
+        To show the figure.
+        """
+        self.figure.show()
+
+    def save(self, fname=None):
+        """
+        To save the figure as fname.
+        :param fname: the filename
+        :type fname: str
+        """
+        if fname is None:
+            fname = get_random_time_stamp()
+        fname = "%s.jpeg" % fname if not fname.endswith(".config") else fname
+        self.figure.savefig(os.path.join(dir_figures, fname), bbox_inches='tight')
+
+    def clear(self):
+        """
+        Clear the figure.
+        """
+        self.figure.clf()
+
+    @staticmethod
+    def get_format(use_marker=False):
+        """
+        Get the format of a line.
+        :param use_marker: whether use markers for points or not.
+        :type use_marker: bool
+        """
+        p_color, p_style, p_marker = 0, 0, 0
+        colors = ['r', 'g', 'b', 'c', 'm', 'y', 'k']
+        styles = ['-', '--', '-.', ':']
+        markers = [""]
+        if use_marker:
+            markers = ['o', 'v', '^', '<', '>', '1', '2', '3', '4', '8', 's', 'p', 'P', '*', 'h', 'H', '+',
+                       'x', 'X', 'D', 'd', '|', '_', ]
+
+        while True:
+            yield colors[p_color] + styles[p_style] + markers[p_marker]
+            p_color += 1
+            p_style += 1
+            p_marker += 1
+            p_color %= len(colors)
+            p_style %= len(styles)
+            p_marker %= len(markers)

zarth_utils-1.0/zarth_utils/general_utils.py

@@ -0,0 +1,26 @@
+import os
+import random
+import datetime
+
+
+def get_datetime():
+    return datetime.datetime.now().strftime('%Y.%m.%d-%H.%M.%S')
+
+
+def get_random_time_stamp():
+    """
+    Return a random time stamp.
+    :return: random time stamp
+    :rtype: str
+    """
+    return "%d-%s" % (random.randint(100, 999), get_datetime())
+
+
+def makedir_if_not_exist(name):
+    """
+    Make the directory if it does not exist.
+    :param name: dir name
+    :type name: str
+    """
+    if not os.path.exists(name):
+        os.makedirs(name)

zarth_utils-1.0/zarth_utils/jupyter_utils.py

@@ -0,0 +1,7 @@
+import pandas as pd
+from IPython.display import display, HTML, Markdown, Latex
+
+display(HTML("<style>.container { width:95% !important; }</style>"))
+pd.set_option('display.max_colwidth', None)
+pd.set_option('display.max_columns', 1000)
+pd.set_option('display.max_rows', 1000)

zarth_utils-1.0/zarth_utils/logger.py

@@ -0,0 +1,45 @@
+import logging
+import sys
+
+from .general_utils import get_random_time_stamp
+
+
+def get_logger(path_log="%s.log" % get_random_time_stamp(), force_add_handler=False):
+    """
+    Set up the logger. Note that the setting will also impact the default logging logger, which means that simply
+    using logging.info() will output the logs to both stdout and the filename_log.
+    :param path_log: the filename of the log
+    :param force_add_handler: if True, will clear logging.root.handlers
+    :type path_log: str
+    """
+    ret_logger = logging.getLogger()
+    ret_logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s-%(name)s-%(levelname)s: %(message)s', datefmt='%Y-%m-%d-%H:%M:%S')
+
+    if force_add_handler:
+        ret_logger.handlers = []
+
+    if not ret_logger.handlers:
+        if path_log is not None:
+            path_log = "%s.log" % path_log if not path_log.endswith(".log") else path_log
+            fh = logging.FileHandler(path_log)
+            fh.setLevel(logging.DEBUG)
+            fh.setFormatter(formatter)
+            ret_logger.addHandler(fh)
+
+        ch = logging.StreamHandler(sys.stdout)
+        ch.setLevel(logging.INFO)
+        ch.setFormatter(formatter)
+        ret_logger.addHandler(ch)
+
+    return ret_logger
+
+
+def logging_info(*args):
+    if logging.root.level > logging.getLevelName("INFO"):
+        logging.root.setLevel(logging.DEBUG)
+        for handler in logging.root.handlers:
+            handler.setLevel(logging.DEBUG)
+        logging.info(*args)
+    else:
+        logging.info(*args)