mindstudio-probe 8.1.2__py3-none-any.whl → 8.2.1__py3-none-any.whl

msprobe/core/compare/multiprocessing_compute.py

@@ -52,16 +52,20 @@ def _ms_graph_handle_multi_process(func, result_df, mode):
 
     def err_call(args):
         logger.error('multiprocess compare failed! Reason: {}'.format(args))
-        try:
-            pool.close()
-        except OSError as e:
-            logger.error(f'pool terminate failed: {str(e)}')
 
     for df_chunk in df_chunks:
         result = pool.apply_async(func, args=(df_chunk, mode), error_callback=err_call)
         results.append(result)
-    final_results = [r.get() for r in results]
+
     pool.close()
+
+    try:
+        final_results = [r.get(timeout=3600) for r in results]
+    except Exception as e:
+        logger.error(f"Task failed with exception: {e}")
+        pool.terminate()
+        raise CompareException(CompareException.MULTIPROCESS_ERROR) from e
+
     pool.join()
     return pd.concat(final_results, ignore_index=True)
 
@@ -94,8 +98,8 @@ class CompareRealData:
     @staticmethod
     def read_dump_data(result_df):
         try:
-            npu_dump_name_list = result_df.iloc[0:, 0].tolist()
-            dump_tensor_pair_list = result_df.iloc[0:, -1].tolist()
+            npu_dump_name_list = result_df.loc[0:, CompareConst.NPU_NAME].tolist()
+            dump_tensor_pair_list = result_df.loc[0:, CompareConst.DATA_NAME].tolist()
             op_name_mapping_dict = {}
             for index, npu_dump_name in enumerate(npu_dump_name_list):
                 dump_tensor_pair = dump_tensor_pair_list[index]
@@ -104,9 +108,9 @@ class CompareRealData:
         except ValueError as e:
             logger.error('result dataframe is not found.')
             raise CompareException(CompareException.INVALID_DATA_ERROR) from e
-        except IndexError as e:
+        except KeyError as e:
             logger.error('result dataframe elements can not be access.')
-            raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e
+            raise CompareException(CompareException.INVALID_KEY_ERROR) from e
 
     @staticmethod
     def _save_cmp_result(offset, result: ComparisonResult, result_df, lock):
@@ -136,7 +140,7 @@ class CompareRealData:
                     result.five_thousand_err_ratio_result)[i]
                 result_df.loc[process_index, CompareConst.ACCURACY] = (
                     check_accuracy(result.cos_result[i], result.max_err_result[i]))
-                result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i]
+                result_df.loc[process_index, CompareConst.ERROR_MESSAGE] += result.err_msgs[i]
             return result_df
         except ValueError as e:
             logger.error('result dataframe is not found.')
@@ -157,20 +161,23 @@ class CompareRealData:
         用于读取excel中的NPU_Name和Bench_Name，根据映射关系找到npy或pt文件，然后读取文件中的数据进行比较，计算余弦相似度、欧式距离
         最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率并生成错误信息
         """
-        error_file, relative_err, error_flag = None, None, False
+        relative_err, error_flag, err_msg = None, False, None
 
         data_name_pair = op_name_mapping_dict.get(npu_op_name)
         npu_data_name = data_name_pair[0]
         bench_data_name = data_name_pair[1]
 
+        error_file = data_name_pair
+
         if str(npu_data_name) == CompareConst.NO_REAL_DATA_FLAG:  # 没有npu真实数据
-            n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True
+            n_value, b_value, error_flag = CompareConst.NO_REAL_DATA, CompareConst.NO_REAL_DATA, True
+            err_msg = "NPU does not have data file."
         elif str(bench_data_name) == CompareConst.NO_REAL_DATA_FLAG:  # 没有bench真实数据
-            n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True
-            error_file = 'no_bench_data'
+            n_value, b_value, error_flag = CompareConst.NO_REAL_DATA, CompareConst.NO_REAL_DATA, True
+            err_msg = "Bench does not have data file."
         elif str(bench_data_name) == CompareConst.N_A:  # bench没匹配
-            n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True
-            error_file = None
+            n_value, b_value, error_flag = CompareConst.API_UNMATCH, CompareConst.API_UNMATCH, True
+            err_msg = "Bench api/module unmatched."
         else:
             npu_dir = input_param.get(CompareConst.NPU_DUMP_DATA_DIR)
             bench_dir = input_param.get(CompareConst.BENCH_DUMP_DATA_DIR)
@@ -187,8 +194,9 @@ class CompareRealData:
                 error_flag = True
 
         # 通过n_value, b_value同时得到错误标志和错误信息
-        n_value, b_value, error_flag, err_msg = get_error_flag_and_msg(n_value, b_value,
-                                                                       error_flag=error_flag, error_file=error_file)
+        if not err_msg:
+            n_value, b_value, error_flag, err_msg = get_error_flag_and_msg(n_value, b_value, error_flag=error_flag,
+                                                                           error_file=error_file)
 
         result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg)
 
@@ -218,11 +226,17 @@ class CompareRealData:
                 = self.compare_by_op(npu_op_name, bench_op_name, dump_path_dict, input_param)
 
             if is_print_compare_log:
-                logger.info(
-                    "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, \
-                    one_thousand_err_ratio {}, "
-                    "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err,
-                                                        err_msg, one_thousand_err_ratio, five_thousand_err_ratio))
+                if "does not have data file" in err_msg:
+                    logger.info(f"[{npu_op_name}] Compare result: {err_msg} ")
+                elif "Bench api/module unmatched" in err_msg:
+                    logger.info(f"[{npu_op_name}] Compare result: {err_msg} ")
+                else:
+                    logger.info(
+                        f"[{npu_op_name}] Compare result: cosine {cos_sim}, euc_dist {euc_dist}, "
+                        f"max_abs_err {max_abs_err}, max_relative_err {max_relative_err}, "
+                        f"one_thousand_err_ratio {one_thousand_err_ratio}, "
+                        f"five_thousand_err_ratio {five_thousand_err_ratio}, {err_msg}"
+                    )
             cos_result.append(cos_sim)
             euc_dist_result.append(euc_dist)
             max_err_result.append(max_abs_err)
@@ -267,10 +281,6 @@ class CompareRealData:
 
         def err_call(args):
             logger.error('multiprocess compare failed! Reason: {}'.format(args))
-            try:
-                pool.close()
-            except OSError:
-                logger.error("pool terminate failed")
 
         progress_bar = tqdm(total=len(result_df), desc="API/Module Item Compare Process", unit="row", ncols=100)
 
@@ -288,7 +298,14 @@ class CompareRealData:
                                       )
             results.append(result)
 
-        final_results = [r.get() for r in results]
         pool.close()
+
+        try:
+            final_results = [r.get(timeout=3600) for r in results]
+        except Exception as e:
+            logger.error(f"Task failed with exception: {e}")
+            pool.terminate()
+            raise CompareException(CompareException.MULTIPROCESS_ERROR) from e
+
         pool.join()
         return pd.concat(final_results, ignore_index=True)

msprobe/core/compare/npy_compare.py

@@ -56,13 +56,7 @@ def get_error_flag_and_msg(n_value, b_value, error_flag=False, error_file=None):
     """判断数据是否有异常并返回异常的n_value, b_value，同时返回error_flag和error_msg"""
     err_msg = ""
     if error_flag:
-        if error_file == "no_bench_data":
-            err_msg = "Bench does not have data file."
-        elif error_file:
-            err_msg = f"Dump file: {error_file} not found or read failed."
-        else:
-            err_msg = CompareConst.NO_BENCH
-        error_flag = True
+        err_msg = f"Dump file: {error_file} not found or read failed."
         return CompareConst.READ_NONE, CompareConst.READ_NONE, error_flag, err_msg
 
     if n_value.size == 0:  # 判断读取到的数据是否为空
@@ -177,8 +171,10 @@ class TensorComparisonBasic(abc.ABC):
 def get_relative_err(n_value, b_value):
     """计算相对误差"""
     with np.errstate(divide='ignore', invalid='ignore'):
+        if n_value.dtype not in CompareConst.FLOAT_TYPE:
+            n_value = n_value.astype(float)
         if b_value.dtype not in CompareConst.FLOAT_TYPE:
-            n_value, b_value = n_value.astype(float), b_value.astype(float)
+            b_value = b_value.astype(float)
 
         n_value_copy = n_value.copy()
         b_value_copy = b_value.copy()
@@ -290,7 +286,8 @@ class CompareOps:
 
 
 def error_value_process(n_value):
-    if n_value in [CompareConst.READ_NONE, CompareConst.UNREADABLE, CompareConst.NONE]:
+    if n_value in [CompareConst.READ_NONE, CompareConst.UNREADABLE, CompareConst.NONE,
+                   CompareConst.NO_REAL_DATA, CompareConst.API_UNMATCH]:
         return CompareConst.UNSUPPORTED, ""
     if n_value == CompareConst.SHAPE_UNMATCH:
         return CompareConst.SHAPE_UNMATCH, ""
@@ -304,7 +301,7 @@ def compare_ops_apply(n_value, b_value, error_flag, err_msg):
     if error_flag:
         result, msg = error_value_process(n_value)
         result_list = [result] * len(CompareOps.compare_ops)
-        err_msg += msg * len(CompareOps.compare_ops)
+        err_msg += msg
         return result_list, err_msg
 
     relative_err = get_relative_err(n_value, b_value)