mindstudio-probe 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

msprobe/pytorch/monitor/unittest/test_monitor.py

@@ -0,0 +1,145 @@
+import os
+import re
+import argparse
+from glob import glob
+
+import pandas as pd
+
+from msprobe.core.common.log import logger
+
+
+def parse_logfile(logfile):
+    grad_norm = []
+    step = []
+    with open(logfile) as f:
+        for line in f.readlines():
+            if 'consumed samples' in line:
+                grad_norm.append(float(re.findall('(?<=grad norm\: )[\d\.]*', line)[0]))
+    return grad_norm
+
+
+def parse_monitor_output(output_dir):
+    reduced = {}
+    unreduced = {}
+    for dir in glob(output_dir + '*'):
+        rank = int(re.findall('(?<=rank)[\d]*', dir)[0])
+        unreduced[rank] = []
+        reduced[rank] = []
+        for file in os.listdir(dir):
+            df = pd.read_csv(os.path.join(dir, file))
+            if '_unreduced_' in file:
+                unreduced[rank].append(df)
+                pass
+            elif '_reduced_' in file:
+                reduced[rank].append(df)
+            else:
+                logger.info(f'unexpected file {file} in {dir}')
+    return reduced, unreduced
+
+
+def valid_reduce(reduced, unreduced, tp_size, dp_size, sequence_parallel):
+    steps = len(reduced[0])
+    world_size = len(reduced)
+    errors = []
+    for index, row in unreduced[0][0].iterrows():
+        param = row['param_name']
+        is_tp_duplicate = False
+        for step in range(2):
+            # sum reduced
+            reduced_mean = 0.
+            for rank in range(world_size):
+                if len(reduced[rank]) == 0:
+                    continue
+                df = reduced[rank][step]
+                value = list(df[df['param_name'] == param]['mean'])
+                if not value:
+                    if step == 0:
+                        is_tp_duplicate = True
+                    continue
+                reduced_mean += value[0]
+
+            # sum unreduced
+            unreduced_mean = 0.
+            for rank in range(world_size):
+                df = unreduced[rank][step]
+                value = list(df[df['param_name'] == param]['mean'])
+                if not value:
+                    continue
+                unreduced_mean += list(df[df['param_name'] == param]['mean'])[0]
+
+            unreduced_mean /= dp_size
+            if is_tp_duplicate and (not sequence_parallel or 'embedding' in param):
+                unreduced_mean /= tp_size
+            try:
+                assert_equal(unreduced_mean, reduced_mean)
+            except AssertionError as e:
+                errors.append([param, step, e, is_tp_duplicate])
+    if errors:
+        logger.info(errors)
+    else:
+        logger.info(f'grad mean is in consist between unreduced grad and reduced grad monitord.')
+
+
+def assert_equal(a, b):
+    if b == 0 or a == 0:
+        return
+    if b == 0:
+        rel_diff = a
+    elif a == 0:
+        rel_diff = b
+    else:
+        rel_diff = abs(a / b - 1)
+    assert rel_diff < 0.01, f'{a}, {b}, {rel_diff}'
+
+
+def valid_total_norm(total_norm, reduced, duplicate_embedding):
+    steps = len(total_norm)
+    world_size = len(reduced)
+    errors = []
+    for step in range(steps):
+        calculated_norm = 0.
+        for rank in range(world_size):
+            if len(reduced[rank]) == 0:
+                if step == 0:
+                    logger.info(f'rank {rank} is duplicated in dp group')
+                continue
+            for index, row in reduced[rank][step].iterrows():
+                if duplicate_embedding and 'word_embedding' in row['param_name']:
+                    continue
+                calculated_norm += row['norm'] ** 2
+        try:
+            assert_equal(calculated_norm ** 0.5, total_norm[step])
+        except AssertionError as e:
+            errors.append([step, e])
+    if errors:
+        logger.info('total norm errors: ', errors)
+    else:
+        logger.info('grad norm in consist between training log and reduced gradients monitored')
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--monitor_output', '-m', type=str, required=True,
+                        help='path prefix to the output of monitor e.g. monitor_output/Aug12_07-16')
+    parser.add_argument('--logfile', '-l', type=str, required=True, help='path to the training log file')
+    parser.add_argument('--tp_size', '-t', type=int, required=True, help='tp parallel size')
+    parser.add_argument('--dp_size', '-d', type=int, required=True, help='dp parallel size')
+    parser.add_argument('--pp_size', '-p', type=int, required=True, help='pp parallel size')
+    parser.add_argument('--untie_embeddings_and_output_weights', '-u', action="store_true", default=False,
+                        help='whether untie_embeddings_and_output_weights in pp parallel')
+    parser.add_argument('--sequence_parallel', '-s', action="store_true", default=False,
+                        help='whether sequence parallel is enabled. Add -s to store true')
+
+    args = parser.parse_args()
+
+    assert args.tp_size > 0, 'if tp not enabled, set tp_size = 1'
+    assert args.dp_size > 0, 'if tp not enabled, set dp_size = 1'
+    assert args.pp_size > 0, 'if tp not enabled, set pp_size = 1'
+
+    total_norm = parse_logfile(args.logfile)
+    reduced, unreduced = parse_monitor_output(args.monitor_output)
+
+    duplicate_embedding = not args.untie_embeddings_and_output_weights and args.pp_size > 1
+
+    valid_total_norm(total_norm, reduced, duplicate_embedding)
+    valid_reduce(reduced, unreduced, args.tp_size, args.dp_size, args.sequence_parallel)

msprobe/pytorch/monitor/utils.py

@@ -0,0 +1,250 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import inspect
+from collections import namedtuple
+from datetime import timezone, timedelta
+from functools import wraps
+
+import torch
+
+from msprobe.core.common.const import MonitorConst, Const
+from msprobe.core.common.log import logger
+from msprobe.core.common.utils import is_int
+
+FILE_MAX_SIZE = 10 * 1024 * 1024 * 1024
+FILE_NAME_MAX_LENGTH = 255
+DIRECTORY_MAX_LENGTH = 4096
+
+beijing_tz = timezone(timedelta(hours=8))
+MVResult = namedtuple('MVResult', ("exp_avg", "exp_avg_sq", "update", "ratio"))
+MVGradResult = namedtuple('MVGradResult', ("exp_avg", "exp_avg_sq", "update", "ratio", "grad"))
+
+
+class MsgConst:
+    """
+    Class for log messages const
+    """
+    SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"]
+
+
+def filter_special_chars(func):
+    @wraps(func)
+    def func_level(msg):
+        for char in MsgConst.SPECIAL_CHAR:
+            msg = msg.replace(char, '_')
+        return func(msg)
+
+    return func_level
+
+
+def get_param_struct(param):
+    res = {}
+    if isinstance(param, (tuple, list)):
+        res['config'] = f'{type(param).__name__}[{len(param)}]'
+        for i, x in enumerate(param):
+            res[i] = f'size={tuple(x.shape)}, dtype={x.dtype}' if torch.is_tensor(x) else f'{type(x)}'
+    elif torch.is_tensor(param):
+        res['config'] = 'tensor'
+        res['tensor'] = f'size={tuple(param.shape)}, dtype={param.dtype}'
+    else:
+        res['config'] = f'{type(param)}'
+        logger.warning(f'Not support type({type(param)}) now, please check the type of param {param}')
+    return res
+
+
+def is_recomputation():
+    """Check if the current operation is in the re-computation phase.
+
+    This function inspects the current call stack to indicate whether the current operation is in the
+    re-computation phase. We use a blacklist mechanism, now supported megatron and mindspeed framework.
+    megatron: The 'backward' function is called by the 'torch/autograd/function.py' file.
+    mindspeed: The 'checkpoint_function_backward' function is called by the 'torch/autograd/function.py'
+    file or the custom module(use CheckpointWithoutOutput) with the 'backward' function is executed within the
+    'torch/_tensor.py' file.
+
+    Returns:
+        bool: True if in the re-computation phase, False otherwise.
+    """
+    backward_function_indices = []
+    call_stack = inspect.stack()
+
+    # Identify the function 'backward' is being executed within the 'torch/_tensor.py' file.
+    for frame_info in call_stack:
+        if frame_info.function == Const.BACKWARD and frame_info.filename.endswith('torch/_tensor.py'):
+            del call_stack
+            return True
+
+    # Identify indices in the call stack where the specific function is being executed
+    for idx, frame_info in enumerate(call_stack):
+        if frame_info.function == Const.BACKWARD or frame_info.function == 'checkpoint_function_backward':
+            backward_function_indices.append(idx)
+
+    # Check if the execution is within 'torch/autograd/function.py' file
+    for idx in backward_function_indices:
+        # The Megatron and MindSpeed L0&L1 scenes
+        if idx + 1 < len(call_stack) and call_stack[idx + 1].filename.endswith('torch/autograd/function.py'):
+            del call_stack
+            return True
+        # The latest MindSpeed L2 and ModelLink scenes
+        if idx + 2 < len(call_stack) and call_stack[idx + 2].filename.endswith('torch/autograd/function.py'):
+            del call_stack
+            return True
+
+    del call_stack
+    return False
+
+
+def validate_ops(ops):
+    if not isinstance(ops, list):
+        raise TypeError("ops should be a list")
+    if not ops:
+        raise TypeError(f"specify ops to calculate metrics. Optional ops: {MonitorConst.OP_LIST}")
+
+    valid_ops = []
+    for op in ops:
+        if op not in MonitorConst.OP_LIST:
+            logger.warning(f"op {op} is not supported. Optional ops: {MonitorConst.OP_LIST}")
+        else:
+            valid_ops.append(op)
+    return valid_ops
+
+
+def validate_ranks(ranks):
+    if not isinstance(ranks, list):
+        raise TypeError("module_ranks should be a list")
+    for rank in ranks:
+        if not isinstance(rank, int) or isinstance(rank, bool):
+            raise TypeError(f"element in module_ranks should be a int, get {type(rank)}")
+
+
+def validate_targets(targets):
+    if not isinstance(targets, dict):
+        raise TypeError('targets in config.json should be a dict')
+    for module_name, field in targets.items():
+        if not isinstance(module_name, str):
+            raise TypeError('key of targets should be module_name[str] in config.json')
+        if not isinstance(field, dict):
+            raise TypeError('values of targets should be cared filed e.g. {"input": "tensor"} in config.json')
+
+
+def validate_print_struct(print_struct):
+    if not isinstance(print_struct, bool):
+        raise TypeError("print_struct should be a bool")
+
+
+def validate_ur_distribution(ur_distribution):
+    if not isinstance(ur_distribution, bool):
+        raise TypeError('ur_distribution should be a bool')
+
+
+def validate_xy_distribution(xy_distribution):
+    if not isinstance(xy_distribution, bool):
+        raise TypeError('xy_distribution should be a bool')
+
+
+def validate_wg_distribution(wg_distribution):
+    if not isinstance(wg_distribution, bool):
+        raise TypeError('wg_distribution should be a bool')
+
+
+def validate_mg_distribution(mg_distribution):
+    if not isinstance(mg_distribution, bool):
+        raise TypeError('mg_distribution should be a bool')
+
+
+def validate_cc_distribution(cc_distribution):
+    if not isinstance(cc_distribution, dict):
+        raise TypeError('cc_distribution should be a dictionary')
+    for key, value in cc_distribution.items():
+        if key == 'enable':
+            if not isinstance(value, bool):
+                raise TypeError('cc_distribution enable should be a bool')
+        elif key == 'cc_codeline':
+            if not isinstance(value, list):
+                raise TypeError('cc_distribution cc_codeline should be a list')
+        elif key == 'cc_pre_hook':
+            if not isinstance(value, bool):
+                raise TypeError('cc_distribution cc_pre_hook should be a bool')
+        elif key == 'cc_log_only':
+            if not isinstance(value, bool):
+                raise TypeError('cc_distribution cc_log_only should be a bool')
+        else:
+            raise TypeError(f'{key} of cc_distribution is not supported.')
+
+
+def validate_alert(alert):
+    if not isinstance(alert, dict):
+        raise TypeError('alert should be a dictionary')
+    rules = alert.get('rules')
+    if rules and isinstance(rules, list):
+        for rule in rules:
+            rule_name = rule.get("rule_name")
+            if rule_name and rule_name not in MonitorConst.RULE_NAME:
+                raise TypeError(f"{rule_name} is not supported")
+            args = rule.get("args")
+            if args and isinstance(args, dict):
+                threshold = args.get("threshold")
+                if not isinstance(threshold, float) or threshold < 0:
+                    raise TypeError('threshold must be float and not less than 0')
+    dump = alert.get('dump')
+    if dump and not isinstance(dump, bool):
+        raise TypeError('dump must be bool.')
+
+
+def validate_step_count_per_record(step_count_per_record):
+    if not is_int(step_count_per_record):
+        raise TypeError('step_count_per_record must be int.')
+    if step_count_per_record < 1:
+        raise ValueError("step_count_per_record must greater than 0")
+    if step_count_per_record > 1e6:
+        raise ValueError("step_count_per_record must smaller than 1e6")
+
+
+def validate_config(config):
+    config['ops'] = validate_ops(config.get('ops', []))
+
+    eps = config.get('eps', 1e-8)
+    if not isinstance(eps, float):
+        raise TypeError("eps should be a float")
+
+    ranks = config.get("module_ranks", [])
+    validate_ranks(ranks)
+
+    targets = config.get("targets", {})
+    validate_targets(targets)
+
+    print_struct = config.get('print_struct', False)
+    validate_print_struct(print_struct)
+
+    ur_distribution = config.get('ur_distribution', False)
+    validate_ur_distribution(ur_distribution)
+
+    xy_distribution = config.get('xy_distribution', False)
+    validate_xy_distribution(xy_distribution)
+
+    wg_distribution = config.get('wg_distribution', False)
+    validate_wg_distribution(wg_distribution)
+
+    mg_distribution = config.get('mg_distribution', False)
+    validate_mg_distribution(mg_distribution)
+
+    cc_distribution = config.get('cc_distribution', {})
+    validate_cc_distribution(cc_distribution)
+
+    alert = config.get('alert', {})
+    validate_alert(alert)
+
+    step_count_per_record = config.get('step_count_per_record', 1)
+    validate_step_count_per_record(step_count_per_record)

msprobe/pytorch/monitor/visualizer.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import torch
+import numpy as np
+import matplotlib.pyplot as plt
+from msprobe.pytorch.monitor.features import cal_histc
+
+
+class HeatmapVisualizer:
+    def __init__(self) -> None:
+        self.histogram_bins_num = 30
+        self.min_val = -1
+        self.max_val = 1
+        self.histogram_edges = None
+        self.histogram_sum_data_np = None  # matrix shape is [bins_num * total_step]
+        self.cur_step_histogram_data = None
+        self.histogram_edges = torch.linspace(self.min_val, self.max_val, self.histogram_bins_num)
+
+    def pre_cal(self, tensor):
+        self.cur_step_histogram_data = cal_histc(tensor_cal=tensor, bins_total=self.histogram_bins_num,
+                                                 min_val=self.min_val, max_val=self.max_val)
+
+    def visualize(self, tag_name: str, step, summary_writer):
+        if self.histogram_sum_data_np is None or self.histogram_sum_data_np.size == 0:
+            self.histogram_sum_data_np = np.expand_dims(self.cur_step_histogram_data.cpu(), 0).T
+        else:
+            # add new data along a different axis because we transposed early
+            # matrix shape is [bins_num * total_step]
+            self.histogram_sum_data_np = np.concatenate((self.histogram_sum_data_np, np.expand_dims(
+                self.cur_step_histogram_data.cpu(), 1)), axis=1)
+
+        fig, ax = plt.subplots()
+        cax = ax.matshow(self.histogram_sum_data_np, cmap='hot', aspect='auto')
+        fig.colorbar(cax)
+
+        lbs = [f'{self.histogram_edges[i]:.2f}' for i in range(self.histogram_bins_num)]
+        plt.yticks(ticks=range(self.histogram_bins_num), labels=lbs)
+        ax.set_xlabel('Step')
+        ax.set_ylabel('Value Range')
+        plt.title(f'Total Step: {step}')
+
+        # Convert matplotlib figure to an image format suitable for TensorBoard
+        fig.canvas.draw()
+        image = torch.from_numpy(np.array(fig.canvas.renderer.buffer_rgba()))
+        plt.close(fig)
+        summary_writer.add_image(tag_name, image.permute(2, 0, 1), global_step=step, dataformats='CHW')

msprobe/pytorch/online_dispatch/__init__.py

@@ -12,9 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+__all__ = ["PtdbgDispatch"]
+
 from signal import signal, SIGPIPE, SIG_DFL
 from .dispatch import PtdbgDispatch
 signal(SIGPIPE, SIG_DFL)
-
-
-__all__ = ["PtdbgDispatch"]

msprobe/pytorch/online_dispatch/compare.py

@@ -1,16 +1,30 @@
-# 进行比对及结果展示
-import os
-import sys
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 import csv
 import json
+import os
+import sys
 from collections import namedtuple
-from rich.table import Table
-from rich.console import Console
+
 from msprobe.core.common.const import CompareConst, FileCheckConst
-from msprobe.core.common.file_utils import FileOpen, change_mode, read_csv
+from msprobe.core.common.file_utils import read_csv, get_json_contents, write_csv
+from msprobe.core.common.utils import check_op_str_pattern_valid
 from msprobe.pytorch.online_dispatch.single_compare import single_benchmark_compare_wrap
-from msprobe.pytorch.common.log import logger
-from msprobe.core.common.utils import CompareException, check_op_str_pattern_valid
+from rich.console import Console
+from rich.table import Table
 
 ELEMENT_NUM_THRESHOLD = 100
 ZERO_NUM_THRESHOLD = 0.1
@@ -19,30 +33,6 @@ FLOAT_PRECISION = 14
 ResultInfo = namedtuple('ResultInfo', ['api_name', 'is_fwd_success', 'is_bwd_success',
                                        'fwd_compare_alg_results', 'bwd_compare_alg_results'])
 
-def get_file_content_bytes(file):
-    with FileOpen(file, 'rb') as file_handle:
-        return file_handle.read()
-
-
-def get_json_contents(file_path):
-    ops = get_file_content_bytes(file_path)
-    try:
-        json_obj = json.loads(ops)
-    except ValueError as error:
-        logger.error('Failed to load "%s". %s' % (file_path, str(error)))
-        raise CompareException(CompareException.INVALID_FILE_ERROR) from error
-    if not isinstance(json_obj, dict):
-        logger.error('Json file %s, content is not a dictionary!' % file_path)
-        raise CompareException(CompareException.INVALID_FILE_ERROR)
-    return json_obj
-
-
-def write_csv(data, filepath):
-    with FileOpen(filepath, 'a', encoding='utf-8-sig') as f:
-        writer = csv.writer(f)
-        writer.writerows(data)
-    change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY)
-
 
 class Saver:
     # consts for result csv
@@ -62,14 +52,15 @@ class Saver:
         }
 
     def write_csv_title(self):
-        summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, self.COLUMN_BACKWARD_SUCCESS, "Message"]]
+        summary_test_rows = [
+            [self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, self.COLUMN_BACKWARD_SUCCESS, "Message"]]
         write_csv(summary_test_rows, self.save_path)
 
         detail_test_rows = [[
             "Npu Name", "Bench Dtype", "NPU Dtype", "Shape",
             "error_balance", "max_abs_diff", "max_abs_idx",
             "max_rel_diff", "max_rel_idx", "eb_thd",
-            "error_thd", "Status","Message"
+            "error_thd", "Status", "Message"
         ]]
         write_csv(detail_test_rows, self.detail_save_path)
 
@@ -106,7 +97,7 @@ class Saver:
         console.print(table_detail)
 
     def get_statistics_from_result_csv(self):
-        checklist = [CompareConst.TRUE, CompareConst.FALSE, CompareConst.NA, CompareConst.SKIP]
+        checklist = [CompareConst.TRUE, CompareConst.FALSE, CompareConst.N_A, CompareConst.SKIP]
         data = read_csv(self.save_path)
         result_csv_name = os.path.basename(self.save_path)
         for _, row in data.iterrows():
@@ -121,7 +112,7 @@ class Saver:
             if column1 == CompareConst.SKIP:
                 continue
             self.test_result_cnt["total_num"] += 1
-            if column1 == CompareConst.TRUE and column2 in [CompareConst.TRUE, 'N/A']:
+            if column1 == CompareConst.TRUE and column2 in [CompareConst.TRUE, CompareConst.N_A]:
                 self.test_result_cnt['success_num'] += 1
             elif column1 == CompareConst.FALSE and column2 == CompareConst.FALSE:
                 self.test_result_cnt['forward_and_backward_fail_num'] += 1
@@ -228,8 +219,8 @@ class Comparator:
             is_bwd_success, bwd_compare_alg_results = True, None
         if is_bwd_success and bwd_compare_alg_results is None:
             self.saver.record_results(ResultInfo(api_name, is_fwd_success, CompareConst.NAN, fwd_compare_alg_results,
-                                      bwd_compare_alg_results))
+                                                 bwd_compare_alg_results))
         else:
             self.saver.record_results(ResultInfo(api_name, is_fwd_success, is_bwd_success, fwd_compare_alg_results,
-                                      bwd_compare_alg_results))
+                                                 bwd_compare_alg_results))
         return is_fwd_success, is_bwd_success