mindstudio-probe 1.0.1__py3-none-any.whl

msprobe/test/pytorch_ut/hook_module/test_wrap_distributed.py

@@ -0,0 +1,35 @@
+import unittest
+import torch.distributed as dist
+from msprobe.pytorch.hook_module.wrap_distributed import *
+
+class TestWrapDistributed(unittest.TestCase):
+    def hook(name, prefix):
+        def forward_pre_hook(nope, input, kwargs):
+            return input, kwargs
+        def forward_hook(nope, input, kwargs, result):
+            return 2
+        def backward_hook():
+            pass
+        return forward_pre_hook, forward_hook, backward_hook
+    
+    def test_get_distributed_ops(self):
+        ops = get_distributed_ops()
+        self.assertIsInstance(ops, set)
+
+    def test_DistributedOPTemplate(self):
+        self.setUp()
+        op_name = 'all_reduce'
+        if op_name in get_distributed_ops():
+            op = DistributedOPTemplate(op_name, self.hook)
+            self.assertEqual(op.op_name_, op_name)
+
+    def test_wrap_distributed_op(self):
+        op_name = 'all_reduce'
+        if op_name in get_distributed_ops():
+            wrapped_op = wrap_distributed_op(op_name, self.hook)
+            self.assertTrue(callable(wrapped_op))
+
+    def test_wrap_distributed_ops_and_bind(self):
+        wrap_distributed_ops_and_bind(self.hook)
+        for op_name in get_distributed_ops():
+            self.assertTrue(hasattr(HOOKDistributedOP, "wrap_" + str(op_name)))

msprobe/test/pytorch_ut/hook_module/test_wrap_functional.py

@@ -0,0 +1,20 @@
+import unittest
+import torch
+from msprobe.pytorch.hook_module import wrap_functional as wf
+
+class TestWrapFunctional(unittest.TestCase):
+
+    def test_remove_dropout(self):
+        input_tensor = torch.randn(20, 16)
+        wf.remove_dropout()
+        output_tensor = torch.nn.functional.dropout(input_tensor)
+        self.assertTrue(torch.equal(input_tensor, output_tensor))
+
+    def test_get_functional_ops(self):
+        expected_ops = {'relu', 'sigmoid', 'softmax'}
+        actual_ops = wf.get_functional_ops()
+        self.assertTrue(expected_ops.issubset(actual_ops))
+
+    def test_wrap_functional_ops_and_bind(self):
+        wf.wrap_functional_ops_and_bind(None)
+        self.assertTrue(hasattr(wf.HOOKFunctionalOP, 'wrap_relu'))

msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py

@@ -0,0 +1,35 @@
+import unittest
+import torch
+import yaml
+from msprobe.pytorch.hook_module.wrap_tensor import get_tensor_ops, HOOKTensor, TensorOPTemplate, wrap_tensor_op, wrap_tensor_ops_and_bind
+
+class TestWrapTensor(unittest.TestCase):
+
+    def hook(name, prefix):
+        def forward_pre_hook(nope, input, kwargs):
+            return input, kwargs
+        def forward_hook(nope, input, kwargs, result):
+            return 2
+        def backward_hook():
+            pass
+        return forward_pre_hook, forward_hook, backward_hook
+    
+    def test_get_tensor_ops(self):
+        result = get_tensor_ops()
+        self.assertIsInstance(result, set)
+
+    def test_HOOKTensor(self):
+        hook_tensor = HOOKTensor()
+        self.assertIsInstance(hook_tensor, HOOKTensor)
+
+    def test_TensorOPTemplate(self):
+        tensor_op_template = TensorOPTemplate('add', self.hook)
+        self.assertTrue(tensor_op_template.op_name_, 'add')
+
+    def test_wrap_tensor_op(self):
+        wrapped_op = wrap_tensor_op('add', self.hook)
+        self.assertTrue(callable(wrapped_op))
+
+    def test_wrap_tensor_ops_and_bind(self):
+        wrap_tensor_ops_and_bind(self.hook)
+        self.assertTrue(hasattr(HOOKTensor, 'wrap_add'))

msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py

@@ -0,0 +1,43 @@
+import unittest
+import torch
+import yaml
+from msprobe.pytorch.hook_module.wrap_torch import *
+
+class TestWrapTorch(unittest.TestCase):
+
+    def hook(name, prefix):
+        def forward_pre_hook(nope, input, kwargs):
+            return input, kwargs
+        def forward_hook(nope, input, kwargs, result):
+            return 2
+        def backward_hook():
+            pass
+        return forward_pre_hook, forward_hook, backward_hook
+    
+    def setUp(self):
+
+        self.op_name = 'add'
+        self.torch_op = wrap_torch_op(self.op_name, self.hook)
+
+    def test_get_torch_ops(self):
+        self.setUp()
+        ops = get_torch_ops()
+        self.assertIsInstance(ops, set)
+        self.assertIn(self.op_name, ops)
+
+    def test_TorchOPTemplate(self):
+        self.setUp()
+        template = TorchOPTemplate(self.op_name, self.hook)
+        self.assertEqual(template.op_name_, self.op_name)        
+        self.assertEqual(template.prefix_op_name_, "Torch." + str(self.op_name) + ".")
+
+    def test_forward(self):
+        self.setUp()
+        template = TorchOPTemplate(self.op_name, self.hook)
+        result = template.forward(torch.tensor([1, 2, 3]), torch.tensor([4, 5, 6]))
+        torch.testing.assert_close(result, torch.tensor([5, 7, 9]))
+
+    def test_wrap_torch_ops_and_bind(self):
+        self.setUp()
+        wrap_torch_ops_and_bind(self.hook)
+        self.assertTrue(hasattr(HOOKTorchOP, "wrap_" + self.op_name))

msprobe/test/pytorch_ut/hook_module/test_wrap_vf.py

@@ -0,0 +1,11 @@
+import unittest
+import torch
+from msprobe.pytorch.hook_module import wrap_vf
+
+class TestWrapVF(unittest.TestCase):
+    def setUp(self):
+        self.hook = lambda x: x
+
+    def test_get_vf_ops(self):
+        ops = wrap_vf.get_vf_ops()
+        self.assertIsInstance(ops, list)

msprobe/test/pytorch_ut/test_pt_config.py

@@ -0,0 +1,69 @@
+from unittest import TestCase
+from unittest.mock import patch, mock_open
+
+from msprobe.core.common.const import Const
+from msprobe.pytorch.pt_config import parse_json_config, parse_task_config
+
+
+class TestPtConfig(TestCase):
+    def test_parse_json_config(self):
+        mock_json_data = {
+            "task": "statistics",
+            "dump_path": "./dump/",
+            "rank": [],
+            "step": [],
+            "level": "L1",
+            "seed": 1234,
+            "statistics": {
+                "scope": [],
+                "list": [],
+                "data_mode": ["all"],
+            },
+            "tensor": {
+                "file_format": "npy"
+            }
+        }
+        with patch("msprobe.pytorch.pt_config.os.path.join", return_value="/path/config.json"), \
+                patch("msprobe.pytorch.pt_config.FileOpen", mock_open(read_data='')), \
+                patch("msprobe.pytorch.pt_config.json.load", return_value=mock_json_data):
+            common_config, task_config = parse_json_config(None, None)
+        self.assertEqual(common_config.task, Const.STATISTICS)
+        self.assertEqual(task_config.data_mode, ["all"])
+
+        with patch("msprobe.pytorch.pt_config.os.path.join", return_value="/path/config.json"), \
+                patch("msprobe.pytorch.pt_config.FileOpen", mock_open(read_data='')), \
+                patch("msprobe.pytorch.pt_config.json.load", return_value=mock_json_data):
+            common_config, task_config = parse_json_config(None, Const.TENSOR)
+        self.assertEqual(common_config.task, Const.STATISTICS)
+        self.assertEqual(task_config.file_format, "npy")
+
+    def test_parse_task_config(self):
+        overflow_check_config = {
+            "overflow_check": {
+                "overflow_nums": 1,
+                "check_mode": "all"
+            }
+        }
+        result = parse_task_config(Const.OVERFLOW_CHECK, overflow_check_config)
+        self.assertEqual(result.overflow_num, 1)
+        self.assertEqual(result.check_mode, "all")
+
+        free_benchmark_config = {
+            "free_benchmark": {
+                "scope": [],
+                "list": ["conv2d"],
+                "fuzz_device": "npu",
+                "pert_mode": "improve_precision",
+                "handler_type": "check",
+                "fuzz_level": "L1",
+                "fuzz_stage": "forward",
+                "if_preheat": False,
+                "preheat_step": 15,
+                "max_sample": 20
+            }
+        }
+        result = parse_task_config(Const.FREE_BENCHMARK, free_benchmark_config)
+        self.assertEqual(result.pert_mode, "improve_precision")
+        self.assertEqual(result.handler_type, "check")
+        self.assertEqual(result.preheat_step, 15)
+        self.assertEqual(result.max_sample, 20)

msprobe/test/pytorch_ut/test_service.py

@@ -0,0 +1,59 @@
+import unittest
+from unittest.mock import patch, mock_open
+
+import torch.nn as nn
+from msprobe.core.common.utils import Const
+from msprobe.pytorch.debugger.debugger_config import DebuggerConfig
+from msprobe.pytorch.pt_config import parse_json_config
+from msprobe.pytorch.service import Service
+
+
+class TestService(unittest.TestCase):
+    def setUp(self):
+        mock_json_data = {
+            "dump_path": "./dump/",
+        }
+        with patch("msprobe.pytorch.pt_config.FileOpen", mock_open(read_data='')), \
+                patch("msprobe.pytorch.pt_config.json.load", return_value=mock_json_data):
+            common_config, task_config = parse_json_config("./config.json", Const.STATISTICS)
+        self.config = DebuggerConfig(common_config, task_config, Const.STATISTICS, "./ut_dump", "L1")
+        self.service = Service(self.config)
+
+    def test_start(self):
+        with patch("msprobe.pytorch.service.get_rank_if_initialized", return_value=0), \
+                patch("msprobe.pytorch.service.Service.create_dirs", return_value=None):
+            self.service.start(None)
+        self.assertEqual(self.service.current_rank, 0)
+
+    def test_stop_and_step(self):
+        with patch("msprobe.core.data_dump.data_collector.DataCollector.write_json", return_value=None):
+            self.service.stop()
+        self.assertFalse(self.service.switch)
+
+        self.service.step()
+        self.assertEqual(self.service.current_iter, 1)
+
+    def test_register_hook_new(self):
+        class TestModule(nn.Module):
+            def __init__(self) -> None:
+                super().__init__()
+                self.linear = nn.Linear(in_features=8, out_features=4)
+
+            def forward(self, x):
+                x = self.linear(x)
+                return x
+
+        self.service.model = TestModule()
+        self.config.level = "L0"
+        with patch("msprobe.pytorch.service.logger.info_on_rank_0") as mock_logger, \
+                patch("msprobe.pytorch.service.remove_dropout", return_value=None):
+            self.service.register_hook_new()
+            self.assertEqual(mock_logger.call_count, 2)
+
+    def test_create_dirs(self):
+        with patch("msprobe.pytorch.service.Path.mkdir", return_value=None), \
+                patch("msprobe.core.common.file_check.FileChecker.common_check", return_value=None), \
+                patch("msprobe.core.data_dump.data_collector.DataCollector.update_dump_paths",
+                      return_value=None):
+            self.service.create_dirs()
+        self.assertEqual(self.service.dump_iter_dir, "./ut_dump/step0")

msprobe/test/resources/advisor.txt

@@ -0,0 +1,3 @@
+Line: NA
+Suspect Nodes: NA
+Expert Advice: All data in comparison result meets the accuracy requirements.

msprobe/test/resources/compare_result_20230703104808.csv

@@ -0,0 +1,9 @@
+NPU Name,Bench Name,NPU Tensor Dtype,Bench Tensor Dtype,NPU Tensor Shape,Bench Tensor Shape,Cosine,MaxAbsErr,NPU max,NPU min,NPU mean,Bench max,Bench min,Bench mean,Accuracy Reached or Not,Err_message
+Functional_linear_0_forward_input.0,Functional_linear_0_forward_input.0,torch.float32,torch.float32,"[3, 2]","[3, 2]",1.0,0.000000,1.948258399963379,-1.0052297115325928,-0.2003595232963562,1.948258399963379,-1.0052297115325928,-0.2003595232963562,Yes,
+Functional_linear_0_forward_input.1,Functional_linear_0_forward_input.1,torch.float32,torch.float32,"[3, 2]","[3, 2]",1.0,0.000000,0.28375449776649475,-0.6661239266395569,-0.2789986729621887,0.28375449776649475,-0.6661239266395569,-0.2789986729621887,Yes,
+Functional_linear_0_forward_input.2,Functional_linear_0_forward_input.2,torch.float32,torch.float32,[3],[3],1.0,0.000000,0.2457989901304245,-0.6338542103767395,-0.14437106251716614,0.2457989901304245,-0.6338542103767395,-0.14437106251716614,Yes,
+Functional_linear_0_forward_output,Functional_linear_0_forward_output,torch.float32,torch.float32,"[3, 3]","[3, 3]",1.0,0.000000,0.8278868794441223,-0.8729169964790344,0.16790540516376495,0.8278868794441223,-0.8729169964790344,0.16790540516376495,Yes,
+Torch_relu_0_forward_input.0,Torch_relu_0_forward_input.0,torch.float32,torch.float32,"[3, 3]","[3, 3]",1.0,0.000000,0.8278868794441223,-0.8729169964790344,0.16790540516376495,0.8278868794441223,-0.8729169964790344,0.16790540516376495,Yes,
+Torch_relu_0_forward_output,Torch_relu_0_forward_output,torch.float32,torch.float32,"[3, 3]","[3, 3]",1.0,0.000000,0.8278868794441223,0.0,0.31367552280426025,0.8278868794441223,0.0,0.31367552280426025,Yes,
+Functional_relu_0_forward_input.0,Functional_relu_0_forward_input.0,torch.float32,torch.float32,"[3, 3]","[3, 3]",1.0,0.000000,0.8278868794441223,-0.8729169964790344,0.16790540516376495,0.8278868794441223,-0.8729169964790344,0.16790540516376495,Yes,
+Functional_relu_0_forward_output,Functional_relu_0_forward_output,torch.float32,torch.float32,"[3, 3]","[3, 3]",1.0,0.000000,0.8278868794441223,0.0,0.31367552280426025,0.8278868794441223,0.0,0.31367552280426025,Yes,

msprobe/test/resources/compare_result_without_accuracy.csv

@@ -0,0 +1,9 @@
+NPU Name,Bench Name,NPU Tensor Dtype,Bench Tensor Dtype,NPU Tensor Shape,Bench Tensor Shape,Cosine,MaxAbsErr,NPU max,NPU min,NPU mean,Bench max,Bench min,Bench mean,Accuracy Reached or Not,Err_message
+,Functional_linear_0_forward_input.0,torch.float32,torch.float32,"[3, 2]","[3, 2]",1,0,1.9482584,-1.005229712,-0.200359523,1.9482584,-1.005229712,-0.200359523,,
+,Functional_linear_0_forward_input.1,torch.float32,torch.float32,"[3, 2]","[3, 2]",1,0,0.283754498,-0.666123927,-0.278998673,0.283754498,-0.666123927,-0.278998673,,
+,Functional_linear_0_forward_input.2,torch.float32,torch.float32,[3],[3],1,0,0.24579899,-0.63385421,-0.144371063,0.24579899,-0.63385421,-0.144371063,,
+,Functional_linear_0_forward_output,torch.float32,torch.float32,"[3, 3]","[3, 3]",1,0,0.827886879,-0.872916996,0.167905405,0.827886879,-0.872916996,0.167905405,,
+,Torch_relu_0_forward_input.0,torch.float32,torch.float32,"[3, 3]","[3, 3]",1,0,0.827886879,-0.872916996,0.167905405,0.827886879,-0.872916996,0.167905405,,
+,Torch_relu_0_forward_output,torch.float32,torch.float32,"[3, 3]","[3, 3]",1,0,0.827886879,0,0.313675523,0.827886879,0,0.313675523,,
+,Functional_relu_0_forward_input.0,torch.float32,torch.float32,"[3, 3]","[3, 3]",1,0,0.827886879,-0.872916996,0.167905405,0.827886879,-0.872916996,0.167905405,,
+,Functional_relu_0_forward_output,torch.float32,torch.float32,"[3, 3]","[3, 3]",1,0,0.827886879,0,0.313675523,0.827886879,0,0.313675523,,

msprobe/test/resources/config.yaml

@@ -0,0 +1,3 @@
+white_list: []
+error_data_path: './'
+precision: 14

msprobe/test/resources/npu_test.pkl

@@ -0,0 +1,8 @@
+["Functional_linear_0_forward_input.0", 1, [], "torch.float32", [3, 2], [1.948258399963379, -1.0052297115325928, -0.2003595232963562]]
+["Functional_linear_0_forward_input.1", 1, [], "torch.float32", [3, 2], [0.28375449776649475, -0.6661239266395569, -0.2789986729621887]]
+["Functional_linear_0_forward_input.2", 1, [], "torch.float32", [3], [0.2457989901304245, -0.6338542103767395, -0.14437106251716614]]
+["Functional_linear_0_forward_output", 1, [], "torch.float32", [3, 3], [0.8278868794441223, -0.8729169964790344, 0.16790540516376495]]
+["Torch_relu_0_forward_input.0", 1, [], "torch.float32", [3, 3], [0.8278868794441223, -0.8729169964790344, 0.16790540516376495]]
+["Torch_relu_0_forward_output", 1, [], "torch.float32", [3, 3], [0.8278868794441223, 0.0, 0.31367552280426025]]
+["Functional_relu_0_forward_input.0", 1, [], "torch.float32", [3, 3], [0.8278868794441223, -0.8729169964790344, 0.16790540516376495]]
+["Functional_relu_0_forward_output", 1, [], "torch.float32", [3, 3], [0.8278868794441223, 0.0, 0.31367552280426025]]

msprobe/test/run_test.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+CUR_DIR=$(dirname $(readlink -f $0))
+TOP_DIR=${CUR_DIR}/..
+TEST_DIR=${TOP_DIR}/"test"
+SRC_DIR=${TOP_DIR}/../
+
+install_pytest() {
+    if ! pip show pytest &> /dev/null; then
+        echo "pytest not found, trying to install..."
+        pip install pytest
+    fi
+
+    if ! pip show pytest-cov &> /dev/null; then
+        echo "pytest-cov not found, trying to install..."
+        pip install pytest-cov
+    fi
+}
+
+run_ut() {
+    install_pytest
+
+    export PYTHONPATH=${SRC_DIR}:${PYTHONPATH}
+    python3 run_ut.py
+}
+
+main() {
+    cd ${TEST_DIR} && run_ut
+}
+
+main $@

msprobe/test/run_ut.py

@@ -0,0 +1,58 @@
+import os
+import shutil
+import subprocess
+import sys
+
+from msprobe.core.common.log import logger
+
+
+def run_ut():
+    cur_dir = os.path.realpath(os.path.dirname(__file__))
+    ut_path = cur_dir
+    cov_dir = os.path.dirname(cur_dir)
+    report_dir = os.path.join(cur_dir, "report")
+    final_xml_path = os.path.join(report_dir, "final.xml")
+    cov_report_path = os.path.join(report_dir, "coverage.xml")
+
+    if os.path.exists(report_dir):
+        shutil.rmtree(report_dir)
+    os.makedirs(report_dir)
+
+    pytest_cmd = [
+                     "python3", "-m", "pytest",
+                     ut_path,
+                     f"--junitxml={final_xml_path}",
+                     f"--cov={cov_dir}",
+                     "--cov-branch",
+                     f"--cov-report=xml:{cov_report_path}",
+                 ]
+
+    try:
+        with subprocess.Popen(
+                pytest_cmd,
+                shell=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                text=True,
+        ) as proc:
+            for line in proc.stdout:
+                logger.info(line.strip())
+
+            proc.wait()
+
+            if proc.returncode == 0:
+                logger.info("Unit tests executed successfully.")
+                return True
+            else:
+                logger.error("Unit tests execution failed.")
+                return False
+    except Exception as e:
+        logger.error(f"An error occurred during test execution: {e}")
+        return False
+
+
+if __name__ == "__main__":
+    if run_ut():
+        sys.exit(0)
+    else:
+        sys.exit(1)

msprobe/test/test_module_processer.py

@@ -0,0 +1,64 @@
+import unittest
+from msprobe.pytorch.module_processer import ModuleProcesser
+from msprobe.pytorch.common.utils import Const
+
+import torch
+
+class TestModuleProcesser(unittest.TestCase):
+    def test_filter_tensor_and_tuple(self):
+        def func(nope, x):
+            return x * 2
+        
+        result_1 = ModuleProcesser.filter_tensor_and_tuple(func)(None, torch.tensor([1]))
+        self.assertEqual(result_1, torch.tensor([2]))
+
+        result_2 = ModuleProcesser.filter_tensor_and_tuple(func)(None, "test")
+        self.assertEqual(result_2, "test")
+
+    def test_clone_return_value_and_test_clone_if_tensor(self):
+        def func(x):
+            return x
+        
+        input = torch.tensor([1])
+        input_tuple = (torch.tensor([1]), torch.tensor([2]))
+        input_list = [torch.tensor([1]), torch.tensor([2])]
+        input_dict = {"A":torch.tensor([1]), "B":torch.tensor([2])}
+
+        result = ModuleProcesser.clone_return_value(func)(input)
+        result[0] = 2
+        self.assertNotEqual(result, input)
+
+        result_tuple = ModuleProcesser.clone_return_value(func)(input_tuple)
+        result_tuple[0][0] = 2
+        self.assertNotEqual(result_tuple, input_tuple)
+
+        result_list = ModuleProcesser.clone_return_value(func)(input_list)
+        result_list[0][0] = 2
+        self.assertNotEqual(result_list, input_list)
+
+        result_dict = ModuleProcesser.clone_return_value(func)(input_dict)
+        result_dict["A"][0] = 2
+        self.assertNotEqual(result_dict, input_dict)
+
+        
+    def test_node_hook(self):
+        empty_list = []
+        test = ModuleProcesser(None)
+        pre_hook = test.node_hook("test", Const.START)
+        self.assertIsNotNone(pre_hook)
+        end_hook = test.node_hook("test", "stop")
+        self.assertIsNotNone(end_hook)
+
+        class A():
+            pass
+        pre_hook(A, None, None)
+        self.assertIn("test", test.module_count)
+        self.assertFalse(test.module_stack==empty_list)
+
+    def test_module_count_func(self):
+        test = ModuleProcesser(None)
+        self.assertEqual(test.module_count, {})
+
+        module_name = "nope"
+        test.module_count_func(module_name)
+        self.assertEqual(test.module_count["nope"], 0)