mindstudio-probe 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

msprobe/core/grad_probe/constant.py

@@ -1,71 +1,75 @@
-
-class GradConst:
-
-    FRAMEWORKS = {"PyTorch", "MindSpore"}
-    PYTORCH = "PyTorch"
-    MindSpore = "MindSpore"
-
-    GRAD_FILE_SUFFIX = {"npy", "pt"}
-    NPY_SUFFIX = "npy"
-    PT_SUFFIX = "pt"
-
-    # for callback
-    CURRENT_STEP = "current_step"
-
-    PARAM_LIST = "param_list"
-    RANK = "rank"
-    STEP = "step"
-    BOUNDS = "bounds"
-    OUTPUT_PATH = "output_path"
-
-    # level const
-    LEVEL = "level"
-    LEVEL0 = "L0"
-    LEVEL1 = "L1"
-    LEVEL2 = "L2"
-    SUPPORTED_LEVEL = {"L0", "L1", "L2"}
-
-    # numpy coding
-    STEP_IDX = 0
-    SHAPE_DIM_IDX = 4
-    MAX_SIZE = 10 * 1024 * 1024 * 1024
-
-    # direction suffix
-    DIR_SUFFIX = "dir.npy"
-
-    # file safty
-    DATA_DIR_AUTHORITY = 0o750
-    DATA_FILE_AUTHORITY = 0o640
-    DIRECTORY_LENGTH = 4096
-    FILE_NAME_LENGTH = 255
-    FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$"
-    PARAM_VALID_PATTERN = r"^[a-zA-Z0-9_.]+$"
-    DIR = "dir"
-    FILE = "file"
-
-    STEP_FINISH = "step_finish"
-
-    SUMMARY = "summary"
-
-    # csv header entry
-    MD5 = "MD5"
-    DISTRIBUTION = "distribution"
-    SHAPE = "shape"
-    MAX = "max"
-    MIN = "min"
-    NORM = "norm"
-
-level_adp = {
-        "L0": {
-            "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
-            "have_grad_direction": False
-        },
-        "L1": {
-            "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
-            "have_grad_direction": True
-        },
-        "L2": {
-            "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
-            "have_grad_direction": True
-        },
+
+class GradConst:
+
+    FRAMEWORKS = {"PyTorch", "MindSpore"}
+    PYTORCH = "PyTorch"
+    MindSpore = "MindSpore"
+
+    GRAD_FILE_SUFFIX = {"npy", "pt"}
+    NPY_SUFFIX = "npy"
+    PT_SUFFIX = "pt"
+
+    # for callback
+    CURRENT_STEP = "current_step"
+
+    PARAM_LIST = "param_list"
+    RANK = "rank"
+    STEP = "step"
+    BOUNDS = "bounds"
+    OUTPUT_PATH = "output_path"
+
+    # level const
+    LEVEL = "level"
+    LEVEL0 = "L0"
+    LEVEL1 = "L1"
+    LEVEL2 = "L2"
+    SUPPORTED_LEVEL = {"L0", "L1", "L2"}
+
+    # numpy coding
+    STEP_IDX = 0
+    SHAPE_DIM_IDX = 4
+    MAX_SIZE = 10 * 1024 * 1024 * 1024
+
+    # direction suffix
+    DIR_SUFFIX = "dir.npy"
+
+    # bounds safety
+    BOUNDS_MINIMUM = -2**63
+    BOUNDS_MAXIMUM = 2**63 - 1
+
+    # file safty
+    DATA_DIR_AUTHORITY = 0o750
+    DATA_FILE_AUTHORITY = 0o640
+    DIRECTORY_LENGTH = 4096
+    FILE_NAME_LENGTH = 255
+    FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$"
+    PARAM_VALID_PATTERN = r"^[a-zA-Z0-9_.]+$"
+    DIR = "dir"
+    FILE = "file"
+
+    STEP_FINISH = "step_finish"
+
+    SUMMARY = "summary"
+
+    # csv header entry
+    MD5 = "MD5"
+    DISTRIBUTION = "distribution"
+    SHAPE = "shape"
+    MAX = "max"
+    MIN = "min"
+    NORM = "norm"
+
+level_adp = {
+        "L0": {
+            "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
+            "have_grad_direction": False
+        },
+        "L1": {
+            "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
+            "have_grad_direction": True
+        },
+        "L2": {
+            "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE],
+            "have_grad_direction": True
+        },
     }

msprobe/core/grad_probe/grad_compare.py

@@ -1,175 +1,170 @@
-import os
-from typing import List
-
-from tqdm import tqdm
-import pandas as pd
-import matplotlib.pyplot as plt
-
-from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create
-from msprobe.core.common.file_check import create_directory
-from msprobe.core.common.log import logger
-from msprobe.core.common.utils import remove_path, write_csv, load_npy
-from msprobe.core.grad_probe.constant import GradConst
-
-
-class GradComparator:
-
-    @staticmethod
-    def _get_grad_weight_order(path1, path2):
-        for summary_file in os.listdir(path1):
-            if not summary_file.endswith(".csv"):
-                continue
-            if not os.path.exists(os.path.join(path2, summary_file)):
-                continue
-            summary_csv = pd.read_csv(os.path.join(path1, summary_file))
-            return summary_csv["param_name"]
-        raise RuntimeError("no matched grad_summary.csv for comparison, please dump data in same configuration")
-    
-    @staticmethod
-    def _get_name_matched_grad_file(param_name, grad_files):
-        for grad_file in grad_files:
-            if param_name == grad_file[:grad_file.rfind('.')]:
-                return grad_file
-        raise RuntimeError("no matched grad_file for comparison, please dump data in same configuration")
-
-    @classmethod
-    def compare_distributed(cls, path1: str, path2: str, output_dir: str):
-        ranks = cls._get_matched_dirs(path1, path2, "rank")
-        logger.info(f"the following ranks will be compared: {ranks}")
-        if not ranks:
-            raise RuntimeError("no matched ranks for comparison, please dump data in same configuration")
-        if not os.path.isdir(output_dir):
-            create_directory(output_dir)
-        for rank in tqdm(ranks, desc="rank"):
-            logger.info(f"now comparing rank {rank}:")
-            cls.compare(os.path.join(path1, f"rank{rank}"),
-                        os.path.join(path2, f"rank{rank}"),
-                        os.path.join(output_dir, f"rank{rank}"))
-
-    @classmethod
-    def compare(cls, path1: str, path2: str, output_dir: str):
-        steps = cls._get_matched_dirs(path1, path2, "step")
-        if not steps:
-            raise RuntimeError("no matched steps for comparison, please dump data in same configuration")
-        similarities = cls._calculate_separated_similarities(path1, path2, steps)
-        if not os.path.isdir(output_dir):
-            create_directory(output_dir)
-        cls._save_similarities(similarities, steps, output_dir)
-
-    @classmethod
-    def _get_matched_dirs(cls, path1: str, path2: str, dir_prefix):
-        check_file_or_directory_path(path1, isdir=True)
-        check_file_or_directory_path(path2, isdir=True)
-        dirs = []
-        for dir_name in os.listdir(path1):
-            index = dir_name.replace(dir_prefix, "", 1)
-            if not dir_name.startswith(dir_prefix) or not index.isdigit():
-                continue
-
-            folder2 = os.path.join(path2, dir_name)
-            if not os.path.isdir(folder2):
-                continue
-            dirs.append(int(index))
-        dirs = sorted(dirs)
-        return dirs
-
-    @classmethod
-    def _save_similarities(cls, similarities: List[float], steps: List[int], output_dir: str):
-        if not similarities:
-            raise ValueError(f"length of similarities is 0")
-        result = [['step'] + [str(step) for step in steps]]
-        for key, value in tqdm(similarities.items(), desc="save similarities (by param)"):
-            if len(value) != len(steps):
-                raise RuntimeError(f"similarities length of {key}:{len(value)} not equal steps:{len(steps)}")
-            plt.plot(steps, value)
-            plt.xlabel('steps')
-            plt.ylabel('similarities')
-            plt.title(f'{key}_similarities')
-            picture_dir = os.path.join(output_dir, "similarities_picture")
-            if not os.path.isdir(picture_dir):
-                create_directory(picture_dir)
-            fig_save_path = os.path.join(picture_dir, f"{key}_similarities.png")
-
-            check_path_before_create(fig_save_path)
-            try:
-                plt.savefig(fig_save_path)
-            except Exception as e:
-                raise RuntimeError(f"save plt figure {fig_save_path} failed") from e
-            plt.close()
-
-            result.append([key] + value)
-        result_csv_path = os.path.join(output_dir, "similarities.csv")
-        if os.path.exists(result_csv_path):
-            logger.warning(f"{result_csv_path} will be recoverd")
-            remove_path(result_csv_path)
-        write_csv(result, result_csv_path)
-
-    @classmethod
-    def _calculate_separated_similarities(cls, path1, path2, steps):
-        similarities = {}
-        logger.info(f"{len(steps)} steps will be compared")
-        grad_weight_order = cls._get_grad_weight_order(path1, path2)
-        for step in tqdm(steps, desc="culculate similarities (by step)"):
-            grad_files = cls._get_matched_grad_files(path1, path2, step)
-            same_count_summary = 0
-            total_count_summary = 0
-            for grad_name in grad_weight_order:
-                grad_file = cls._get_name_matched_grad_file(grad_name, grad_files)
-                grad1 = os.path.join(path1, f"step{step}", grad_file)
-                grad2 = os.path.join(path2, f"step{step}", grad_file)
-                same_count, total_count = cls._calculate_similarity(grad1, grad2)
-                same_count_summary += same_count
-                total_count_summary += total_count
-                idx = grad_file.rfind(".")
-                param_name = grad_file[:idx]
-                if param_name not in similarities:
-                    similarities[param_name] = []
-                if total_count == 0:
-                    similarities[param_name].append(0)
-                else:
-                    similarities[param_name].append(same_count / total_count)
-            if GradConst.SUMMARY not in similarities:
-                similarities[GradConst.SUMMARY] = []
-            if total_count_summary == 0:
-                similarities[GradConst.SUMMARY].append(0)
-            else:
-                similarities[GradConst.SUMMARY].append(same_count_summary / total_count_summary)
-        return similarities
-
-    @classmethod
-    def _get_matched_grad_files(cls, path1: str, path2: str, step: int):
-        path1 = os.path.join(path1, f"step{step}")
-        path2 = os.path.join(path2, f"step{step}")
-        check_file_or_directory_path(path1, isdir=True)
-        check_file_or_directory_path(path2, isdir=True)
-        grad_files = []
-        for grad_file in os.listdir(path1):
-            splits = grad_file.split('.')
-            if len(splits) < 1 or splits[-1] not in GradConst.GRAD_FILE_SUFFIX:
-                continue
-            folder2 = os.path.join(path2, grad_file)
-            if not os.path.exists(folder2):
-                continue
-            grad_files.append(grad_file)
-        return sorted(grad_files)
-
-    @classmethod
-    def _calculate_similarity(cls, grad_file1: str, grad_file2: str):
-        npy1, npy2 = cls._load_grad_files(grad_file1, grad_file2)
-        same_count = (npy1 == npy2).sum()
-        total_count = npy1.size
-        return same_count, total_count
-
-    @classmethod
-    def _load_grad_files(cls, grad_file1: str, grad_file2: str):
-        grad1 = load_npy(grad_file1)
-        grad2 = load_npy(grad_file2)
-        if grad1.shape != grad2.shape:
-            raise RuntimeError(f"tensor shape is not equal: {grad_file1}, {grad_file2}")
-        if grad1.dtype != bool:
-            raise TypeError(f"tensor type is not bool: {grad_file1}")
-        if grad2.dtype != bool:
-            raise TypeError(f"tensor type is not bool: {grad_file2}")
-        return grad1, grad2
-
-
+import os
+from typing import List
+
+from tqdm import tqdm
+import matplotlib.pyplot as plt
+
+from msprobe.core.common.file_utils import create_directory, check_path_before_create, check_file_or_directory_path
+from msprobe.core.common.log import logger
+from msprobe.core.common.file_utils import remove_path, load_npy, write_csv, read_csv
+from msprobe.core.grad_probe.constant import GradConst
+from msprobe.core.grad_probe.utils import plt_savefig
+
+
+class GradComparator:
+
+    @staticmethod
+    def _get_grad_weight_order(path1, path2):
+        for summary_file in os.listdir(path1):
+            if not summary_file.endswith(".csv"):
+                continue
+            if not os.path.exists(os.path.join(path2, summary_file)):
+                continue
+            summary_csv = read_csv(os.path.join(path1, summary_file))
+            return summary_csv["param_name"]
+        raise RuntimeError("no matched grad_summary.csv for comparison, please dump data in same configuration")
+    
+    @staticmethod
+    def _get_name_matched_grad_file(param_name, grad_files):
+        for grad_file in grad_files:
+            if param_name == grad_file[:grad_file.rfind('.')]:
+                return grad_file
+        raise RuntimeError("no matched grad_file for comparison, please dump data in same configuration")
+
+    @classmethod
+    def compare_distributed(cls, path1: str, path2: str, output_dir: str):
+        ranks = cls._get_matched_dirs(path1, path2, "rank")
+        logger.info(f"the following ranks will be compared: {ranks}")
+        if not ranks:
+            raise RuntimeError("no matched ranks for comparison, please dump data in same configuration")
+        if not os.path.isdir(output_dir):
+            create_directory(output_dir)
+        for rank in tqdm(ranks, desc="rank"):
+            logger.info(f"now comparing rank {rank}:")
+            cls.compare(os.path.join(path1, f"rank{rank}"),
+                        os.path.join(path2, f"rank{rank}"),
+                        os.path.join(output_dir, f"rank{rank}"))
+
+    @classmethod
+    def compare(cls, path1: str, path2: str, output_dir: str):
+        steps = cls._get_matched_dirs(path1, path2, "step")
+        if not steps:
+            raise RuntimeError("no matched steps for comparison, please dump data in same configuration")
+        similarities = cls._calculate_separated_similarities(path1, path2, steps)
+        if not os.path.isdir(output_dir):
+            create_directory(output_dir)
+        cls._save_similarities(similarities, steps, output_dir)
+
+    @classmethod
+    def _get_matched_dirs(cls, path1: str, path2: str, dir_prefix):
+        check_file_or_directory_path(path1, isdir=True)
+        check_file_or_directory_path(path2, isdir=True)
+        dirs = []
+        for dir_name in os.listdir(path1):
+            index = dir_name.replace(dir_prefix, "", 1)
+            if not dir_name.startswith(dir_prefix) or not index.isdigit():
+                continue
+
+            folder2 = os.path.join(path2, dir_name)
+            if not os.path.isdir(folder2):
+                continue
+            dirs.append(int(index))
+        dirs = sorted(dirs)
+        return dirs
+
+    @classmethod
+    def _save_similarities(cls, similarities: List[float], steps: List[int], output_dir: str):
+        if not similarities:
+            raise ValueError(f"length of similarities is 0")
+        result = [['step'] + [str(step) for step in steps]]
+        for key, value in tqdm(similarities.items(), desc="save similarities (by param)"):
+            if len(value) != len(steps):
+                raise RuntimeError(f"similarities length of {key}:{len(value)} not equal steps:{len(steps)}")
+            plt.plot(steps, value)
+            plt.xlabel('steps')
+            plt.ylabel('similarities')
+            plt.title(f'{key}_similarities')
+            picture_dir = os.path.join(output_dir, "similarities_picture")
+            if not os.path.isdir(picture_dir):
+                create_directory(picture_dir)
+            fig_save_path = os.path.join(picture_dir, f"{key}_similarities.png")
+
+            plt_savefig(fig_save_path)
+            plt.close()
+
+            result.append([key] + value)
+        result_csv_path = os.path.join(output_dir, "similarities.csv")
+        if os.path.exists(result_csv_path):
+            logger.warning(f"{result_csv_path} will be recoverd")
+            remove_path(result_csv_path)
+        write_csv(result, result_csv_path)
+
+    @classmethod
+    def _calculate_separated_similarities(cls, path1, path2, steps):
+        similarities = {}
+        logger.info(f"{len(steps)} steps will be compared")
+        grad_weight_order = cls._get_grad_weight_order(path1, path2)
+        for step in tqdm(steps, desc="culculate similarities (by step)"):
+            grad_files = cls._get_matched_grad_files(path1, path2, step)
+            same_count_summary = 0
+            total_count_summary = 0
+            for grad_name in grad_weight_order:
+                grad_file = cls._get_name_matched_grad_file(grad_name, grad_files)
+                grad1 = os.path.join(path1, f"step{step}", grad_file)
+                grad2 = os.path.join(path2, f"step{step}", grad_file)
+                same_count, total_count = cls._calculate_similarity(grad1, grad2)
+                same_count_summary += same_count
+                total_count_summary += total_count
+                idx = grad_file.rfind(".")
+                param_name = grad_file[:idx]
+                if param_name not in similarities:
+                    similarities[param_name] = []
+                if total_count == 0:
+                    similarities[param_name].append(0)
+                else:
+                    similarities[param_name].append(same_count / total_count)
+            if GradConst.SUMMARY not in similarities:
+                similarities[GradConst.SUMMARY] = []
+            if total_count_summary == 0:
+                similarities[GradConst.SUMMARY].append(0)
+            else:
+                similarities[GradConst.SUMMARY].append(same_count_summary / total_count_summary)
+        return similarities
+
+    @classmethod
+    def _get_matched_grad_files(cls, path1: str, path2: str, step: int):
+        path1 = os.path.join(path1, f"step{step}")
+        path2 = os.path.join(path2, f"step{step}")
+        check_file_or_directory_path(path1, isdir=True)
+        check_file_or_directory_path(path2, isdir=True)
+        grad_files = []
+        for grad_file in os.listdir(path1):
+            splits = grad_file.split('.')
+            if len(splits) < 1 or splits[-1] not in GradConst.GRAD_FILE_SUFFIX:
+                continue
+            folder2 = os.path.join(path2, grad_file)
+            if not os.path.exists(folder2):
+                continue
+            grad_files.append(grad_file)
+        return sorted(grad_files)
+
+    @classmethod
+    def _calculate_similarity(cls, grad_file1: str, grad_file2: str):
+        npy1, npy2 = cls._load_grad_files(grad_file1, grad_file2)
+        same_count = (npy1 == npy2).sum()
+        total_count = npy1.size
+        return same_count, total_count
+
+    @classmethod
+    def _load_grad_files(cls, grad_file1: str, grad_file2: str):
+        grad1 = load_npy(grad_file1)
+        grad2 = load_npy(grad_file2)
+        if grad1.shape != grad2.shape:
+            raise RuntimeError(f"tensor shape is not equal: {grad_file1}, {grad_file2}")
+        if grad1.dtype != bool:
+            raise TypeError(f"tensor type is not bool: {grad_file1}")
+        if grad2.dtype != bool:
+            raise TypeError(f"tensor type is not bool: {grad_file2}")
+        return grad1, grad2
+
+

msprobe/core/grad_probe/utils.py

@@ -1,52 +1,77 @@
-import re
-from msprobe.core.grad_probe.constant import GradConst
-from msprobe.core.common.log import logger
-from msprobe.core.common.utils import write_csv
-
-def data_in_list_target(data, lst):
-    return not lst or len(lst) == 0 or data in lst
-
-
-def check_numeral_list_ascend(lst):
-    if any(not isinstance(item, (int, float)) for item in lst):
-        raise Exception("The input list should only contain numbers")
-    if lst != sorted(lst):
-        raise Exception("The input list should be ascending")
-
-
-def check_param(param_name):
-    if not re.match(GradConst.PARAM_VALID_PATTERN, param_name):
-        raise RuntimeError("The parameter name contains special characters.")
-    
-
-def check_str(string, variable_name):
-    if not isinstance(string, str):
-        raise ValueError(f'The variable: "{variable_name}" is not a string.')
-    
-
-class ListCache(list):
-    threshold = 1000
-
-    def __init__(self, *args):
-        super().__init__(*args)
-        self._output_file = None
-
-    def __del__(self):
-        self.flush()
-
-    def flush(self):
-        if len(self) == 0:
-            return
-        if not self._output_file:
-            logger.warning("dumpfile path is not setted")
-        write_csv(self, self._output_file)
-        logger.info(f"write {len(self)} items to {self._output_file}.")
-        self.clear()
-
-    def append(self, data):
-        list.append(self, data)
-        if len(self) >= ListCache.threshold:
-            self.flush()
-    
-    def set_output_file(self, output_file):
-        self._output_file = output_file
+import re
+from msprobe.core.grad_probe.constant import GradConst
+from msprobe.core.common.log import logger
+from msprobe.core.common.file_utils import write_csv, check_path_before_create, change_mode
+from msprobe.core.common.const import FileCheckConst
+import matplotlib.pyplot as plt
+
+
+def data_in_list_target(data, lst):
+    return not lst or len(lst) == 0 or data in lst
+
+
+def check_numeral_list_ascend(lst):
+    if any(not isinstance(item, (int, float)) for item in lst):
+        raise Exception("The input list should only contain numbers")
+    if lst != sorted(lst):
+        raise Exception("The input list should be ascending")
+
+
+def check_param(param_name):
+    if not re.match(GradConst.PARAM_VALID_PATTERN, param_name):
+        raise RuntimeError("The parameter name contains special characters.")
+
+
+def check_str(string, variable_name):
+    if not isinstance(string, str):
+        raise ValueError(f'The variable: "{variable_name}" is not a string.')
+
+def check_bounds_element(bound):
+    return GradConst.BOUNDS_MINIMUM <= bound and bound <= GradConst.BOUNDS_MAXIMUM
+
+def check_bounds(bounds):
+    prev = GradConst.BOUNDS_MINIMUM - 1
+    for element in bounds:
+        if not isinstance(element, (int, float)):
+            raise Exception("bounds element is not int or float")
+        if not check_bounds_element(element):
+            raise Exception("bounds element is out of int64 range")
+        if prev >= element:
+            raise Exception("bounds list is not ascending")
+        prev = element
+
+class ListCache(list):
+    threshold = 1000
+
+    def __init__(self, *args):
+        super().__init__(*args)
+        self._output_file = None
+
+    def __del__(self):
+        self.flush()
+
+    def flush(self):
+        if len(self) == 0:
+            return
+        if not self._output_file:
+            logger.warning("dumpfile path is not setted")
+        write_csv(self, self._output_file)
+        logger.info(f"write {len(self)} items to {self._output_file}.")
+        self.clear()
+
+    def append(self, data):
+        list.append(self, data)
+        if len(self) >= ListCache.threshold:
+            self.flush()
+
+    def set_output_file(self, output_file):
+        self._output_file = output_file
+
+
+def plt_savefig(fig_save_path):
+    check_path_before_create(fig_save_path)
+    try:
+        plt.savefig(fig_save_path)
+    except Exception as e:
+        raise RuntimeError(f"save plt figure {fig_save_path} failed") from e
+    change_mode(fig_save_path, FileCheckConst.DATA_FILE_AUTHORITY)