mindstudio-probe 1.0.1__py3-none-any.whl → 1.0.3__py3-none-any.whl

msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py

@@ -2,13 +2,11 @@ import time
 import os
 import math
 
-import numpy as np
 import torch
-import yaml
-from msprobe.core.common.utils import CompareException
+
+from msprobe.core.common.utils import CompareException, load_yaml
 from msprobe.core.common.const import Const
 from msprobe.pytorch.common.log import logger
-from msprobe.core.common.file_check import FileOpen
 
 
 current_time = time.strftime("%Y%m%d%H%M%S")
@@ -22,17 +20,15 @@ BINARY_COMPARE_UNSUPPORT_LIST = BENCHMARK_COMPARE_SUPPORT_LIST + API_PRECISION_C
 
 cur_path = os.path.dirname(os.path.realpath(__file__))
 standard_yaml_path = os.path.join(cur_path, "api_precision_standard.yaml")
-with FileOpen(standard_yaml_path, 'r') as f:
-    Apis = yaml.safe_load(f)
-    AbsoluteStandardApi = Apis.get('AbsoluteThreshStandard')
-    BinaryStandardApi = Apis.get('BinaryCompareStandard')
-    ULPStandardApi = Apis.get('ULPStandard')
-    ThousandthStandardApi = Apis.get('ThousandthStandard')
+apis = load_yaml(standard_yaml_path)
+absolute_standard_api = apis.get('AbsoluteThreshStandard')
+binary_standard_api = apis.get('BinaryCompareStandard')
+ulp_standard_api = apis.get('ULPStandard')
+thousandth_standard_api = apis.get('ThousandthStandard')
 
 
 threshold_yaml_path = os.path.join(cur_path, "api_precision_threshold.yaml")
-with FileOpen(threshold_yaml_path, 'r') as f:
-    apis_threshold = yaml.safe_load(f)
+apis_threshold = load_yaml(threshold_yaml_path)
 
 
 DETAIL_TEST_ROWS = [[

msprobe/pytorch/api_accuracy_checker/config.yaml

@@ -1,4 +1,10 @@
 white_list: []
+black_list: []
 error_data_path: './'
 precision: 14
- 
+is_online: False
+nfs_path: ""
+host: ""
+port: -1
+rank_list: [0]
+tls_path: ""

msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py

@@ -21,10 +21,11 @@ import torch
 import numpy
 
 from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import hf_32_standard_api
-from msprobe.pytorch.api_accuracy_checker.common.utils import check_file_or_directory_path, check_object_type, \
-    get_full_data_path, CompareException
+from msprobe.pytorch.api_accuracy_checker.common.utils import check_object_type, get_full_data_path, \
+    CompareException
+from msprobe.core.common.file_check import FileChecker
 from msprobe.pytorch.common.log import logger
-from msprobe.core.common.const import Const
+from msprobe.core.common.const import Const, FileCheckConst
 
 TORCH_TYPE = ["torch.device", "torch.dtype"]
 TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"]
@@ -87,12 +88,13 @@ def gen_real_tensor(data_path, convert_type):
         convert_type: convert ori_type to dist_type flag.
     """
     data_path = os.path.realpath(data_path)
-    check_file_or_directory_path(data_path)
+    data_path_checker = FileChecker(data_path, FileCheckConst.FILE, ability=FileCheckConst.READ_ABLE)
+    data_path = data_path_checker.common_check()
     if not data_path.endswith('.pt') and not data_path.endswith('.npy'):
         error_info = f"The file: {data_path} is not a pt or numpy file."
         raise CompareException(CompareException.INVALID_FILE_ERROR, error_info)
     if data_path.endswith('.pt'):
-        data = torch.load(data_path).cpu()
+        data = torch.load(data_path, map_location=torch.device('cpu'))
     else:
         data_np = numpy.load(data_path)
         data = torch.from_numpy(data_np)
@@ -255,12 +257,13 @@ def gen_args(args_info, api_name, need_grad=True, convert_type=None, real_data_p
     return args_result
 
 
-def gen_kwargs(api_info, convert_type=None, real_data_path=None):
+def gen_kwargs(api_info, api_name, convert_type=None, real_data_path=None):
     """
     Function Description:
         Based on API basic information, generate input parameters: kwargs, for API forward running
     Parameter:
         api_info: API basic information. Dict
+        api_name: API name
         convert_type: convert ori_type to dist_type flag.
         real_data_path: the root directory for storing real data.
     """
@@ -268,11 +271,11 @@ def gen_kwargs(api_info, convert_type=None, real_data_path=None):
     kwargs_params = api_info.get("input_kwargs")
     for key, value in kwargs_params.items():
         if isinstance(value, (list, tuple)):
-            kwargs_params[key] = gen_list_kwargs(value, convert_type, real_data_path)
+            kwargs_params[key] = gen_list_kwargs(value, api_name, convert_type, real_data_path)
         elif value is None:
             kwargs_params[key] = None
         elif value.get('type') in TENSOR_DATA_LIST or value.get('type').startswith("numpy"):
-            kwargs_params[key] = gen_data(value, True, convert_type, real_data_path)
+            kwargs_params[key] = gen_data(value, api_name, True, convert_type, real_data_path)
         elif value.get('type') in TORCH_TYPE:
             gen_torch_kwargs(kwargs_params, key, value)
         else:
@@ -285,18 +288,19 @@ def gen_torch_kwargs(kwargs_params, key, value):
         kwargs_params[key] = eval(value.get('value'))
 
 
-def gen_list_kwargs(kwargs_item_value, convert_type, real_data_path=None):
+def gen_list_kwargs(kwargs_item_value, api_name, convert_type, real_data_path=None):
     """
     Function Description:
         When kwargs value is list, generate the list of kwargs result
     Parameter:
         kwargs_item_value: kwargs value before to generate. List
+        api_name: API name
         convert_type: convert ori_type to dist_type flag.
     """
     kwargs_item_result = []
     for item in kwargs_item_value:
         if item.get('type') in TENSOR_DATA_LIST:
-            item_value = gen_data(item, False, convert_type, real_data_path)
+            item_value = gen_data(item, api_name, False, convert_type, real_data_path)
         elif item.get('type') == "torch.Size":
             item_value = torch.Size(item.get('value'))
         else:
@@ -319,7 +323,7 @@ def gen_api_params(api_info, api_name, need_grad=True, convert_type=None, real_d
     if convert_type and convert_type not in Const.CONVERT:
         error_info = f"convert_type params not support {convert_type}."
         raise CompareException(CompareException.INVALID_PARAM_ERROR, error_info)
-    kwargs_params = gen_kwargs(api_info, convert_type, real_data_path)
+    kwargs_params = gen_kwargs(api_info, api_name, convert_type, real_data_path)
     if api_info.get("input_args"):
         args_params = gen_args(api_info.get("input_args"), api_name, need_grad, convert_type, real_data_path)
     else:

msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py

@@ -9,8 +9,9 @@ import threading
 from collections import namedtuple
 from itertools import cycle
 from tqdm import tqdm
-from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, get_validated_result_csv_path, \
-    get_validated_details_csv_path, preprocess_forward_content
+from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, preprocess_forward_content
+from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validated_result_csv_path, \
+    get_validated_details_csv_path
 from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator
 from msprobe.pytorch.common import parse_json_info_forward_backward
 from msprobe.core.common.file_check import FileChecker, check_file_suffix, check_link, FileOpen, \
@@ -68,7 +69,7 @@ signal.signal(signal.SIGTERM, signal_handler)
 
 ParallelUTConfig = namedtuple('ParallelUTConfig', ['api_files', 'out_path', 'num_splits',
                                                    'save_error_data_flag', 'jit_compile_flag', 'device_id',
-                                                   'result_csv_path', 'total_items', 'real_data_path'])
+                                                   'result_csv_path', 'total_items', 'config_path'])
 
 
 def run_parallel_ut(config):
@@ -90,7 +91,7 @@ def run_parallel_ut(config):
             *(['-j'] if config.jit_compile_flag else []),
             *(['-save_error_data'] if config.save_error_data_flag else []),
             '-csv_path', config.result_csv_path,
-            *(['-real_data_path', config.real_data_path] if config.real_data_path else [])
+            *(['-config', config.config_path] if config.config_path else [])
         ]
         return cmd
 
@@ -110,19 +111,14 @@ def run_parallel_ut(config):
 
     def update_progress_bar(progress_bar, result_csv_path):
         while any(process.poll() is None for process in processes):
-            try:
-                with open(result_csv_path, 'r') as result_file:
-                    completed_items = len(result_file.readlines()) - 1
-                    progress_bar.update(completed_items - progress_bar.n)
-            except FileNotFoundError:
-                logger.warning(f"Result CSV file not found: {result_csv_path}.")
-            except Exception as e:
-                logger.error(f"An unexpected error occurred while reading result CSV: {e}")
+            with FileOpen(result_csv_path, 'r') as result_file:
+                completed_items = len(result_file.readlines()) - 1
+                progress_bar.update(completed_items - progress_bar.n)
             time.sleep(1)
 
     for api_info in config.api_files:
         cmd = create_cmd(api_info, next(device_id_cycle))
-        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1)
+        process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1, shell=False)
         processes.append(process)
         threading.Thread(target=read_process_output, args=(process,), daemon=True).start()
 
@@ -175,7 +171,7 @@ def prepare_config(args):
     out_path_checker = FileChecker(out_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE)
     out_path = out_path_checker.common_check()
     split_files, total_items = split_json_file(api_info, args.num_splits, args.filter_api)
-
+    config_path = os.path.realpath(args.config_path) if args.config_path else None
     result_csv_path = args.result_csv_path or os.path.join(out_path, f"accuracy_checking_result_{time.strftime('%Y%m%d%H%M%S')}.csv")
     if not args.result_csv_path:
         details_csv_path = os.path.join(out_path, f"accuracy_checking_details_{time.strftime('%Y%m%d%H%M%S')}.csv")
@@ -187,7 +183,7 @@ def prepare_config(args):
     logger.info(f"UT task details will be saved in {details_csv_path}")
     return ParallelUTConfig(split_files, out_path, args.num_splits, args.save_error_data,
                             args.jit_compile, args.device_id, result_csv_path,
-                            total_items, args.real_data_path)
+                            total_items, config_path)
 
 
 def main():

msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py

@@ -10,10 +10,14 @@ else:
     is_gpu = False
 import torch
 from tqdm import tqdm
-from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import exec_api, generate_device_params, get_api_info
-from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents
+from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import generate_device_params, get_api_info
+from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import exec_api
+from msprobe.core.common.utils import get_json_contents
 from msprobe.core.common.file_check import check_link
 from msprobe.pytorch.common.log import logger
+from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward
+from msprobe.core.common.const import Const
+
 
 def check_tensor_overflow(x):
     if isinstance(x, torch.Tensor) and x.numel() != 0 and x.dtype != torch.bool:
@@ -52,12 +56,12 @@ def check_data_overflow(x):
 
 def run_overflow_check(forward_file):
     logger.info("start UT test")
-    forward_content = get_json_contents(forward_file)
+    forward_content, _, real_data_path = parse_json_info_forward_backward(forward_file)
     for api_full_name, api_info_dict in tqdm(forward_content.items()):
         try:
-            run_torch_api(api_full_name, api_info_dict)
+            run_torch_api(api_full_name, api_info_dict, real_data_path)
         except Exception as err:
-            api_name = api_full_name.split("_", 1)[1].rsplit("_", 2)[0]
+            _, api_name, _ = api_full_name.split(Const.SEP)
             if "not implemented for 'Half'" in str(err):
                 logger.warning(f"API {api_name} not support half tensor in CPU, please add {api_name} to CONVERT_API "
                                f"'fp16_to_fp32' list in accuracy_tools/api_accuracy_check/common/utils.py file.")
@@ -68,11 +72,10 @@ def run_overflow_check(forward_file):
                 logger.error(f"Run {api_full_name} UT Error: %s" % str(err))
 
 
-def run_torch_api(api_full_name, api_info_dict):
+def run_torch_api(api_full_name, api_info_dict, real_data_path):
     torch.npu.clear_npu_overflow_flag()
-    api_type = api_full_name.split(".")[0]
-    api_name = api_full_name.split(".", 1)[1].rsplit(".", 2)[0]
-    args, kwargs, need_grad = get_api_info(api_info_dict, api_name, real_data_path='')
+    api_type, api_name, _ = api_full_name.split(Const.SEP)
+    args, kwargs, need_grad = get_api_info(api_info_dict, api_name, real_data_path)
     if not need_grad:
         logger.warning("%s function with out=... arguments don't support automatic differentiation, skip backward." 
                        % api_full_name)
@@ -81,6 +84,10 @@ def run_torch_api(api_full_name, api_info_dict):
         del kwargs["device"]
     out = exec_api(api_type, api_name, args, kwargs)
     npu_out = exec_api(api_type, api_name, npu_args, npu_kwargs)
+    if out is None and npu_out is None:
+        logger.warning("The %s overflow is a normal overflow, out and npu_out is None." % api_full_name)
+        return
+
     cpu_overflow = check_data_overflow(out)
     npu_overflow = torch_npu.npu.utils.npu_check_overflow(npu_out)
     if cpu_overflow == npu_overflow: