mindstudio-probe 1.2.2__py3-none-any.whl → 1.3.0__py3-none-any.whl

msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml

@@ -564,15 +564,15 @@ tensor:
   - all
   - amax
   - amin
+  - angle
   - any
   - arccos
   - arccosh
-  - argmax
-  - angle
   - arcsin
   - arcsinh
   - arctan
   - arctanh
+  - argmax
   - argmin
   - argsort
   - asin
@@ -582,19 +582,23 @@ tensor:
   - atanh
   - baddbmm
   - bernoulli
+  - bfloat16
   - bincount
   - bitwise_and
   - bitwise_or
   - bitwise_xor
   - bmm
   - bool
+  - bool astype
   - broadcast_to
+  - byte
   - ceil
-  - cholesky_solve
   - cholesky
+  - cholesky_solve
   - clamp
   - clip
   - conj
+  - copy
   - copysign
   - cos
   - cosh
@@ -606,11 +610,13 @@ tensor:
   - deg2rad
   - diag
   - diagflat
+  - diagonal
   - diff
   - digamma
   - div
   - div_
   - divide
+  - double
   - equal
   - erf
   - erfc
@@ -618,13 +624,16 @@ tensor:
   - exp
   - expand_as
   - expm1
+  - flatten
   - flip
   - fliplr
   - flipud
+  - float
   - float_power
   - floor
   - fmod
   - frac
+  - from_numpy
   - gather_elements
   - ge
   - geqrf
@@ -648,12 +657,12 @@ tensor:
   - inner
   - int
   - inverse
+  - is_complex
+  - is_signed
   - isclose
   - isfinite
   - isinf
   - isnan
-  - is_complex
-  - is_signed
   - isneginf
   - isposinf
   - isreal
@@ -704,28 +713,27 @@ tensor:
   - new_ones
   - new_zeros
   - nextafter
-  - norm
   - nonzero
+  - norm
   - not_equal
   - ormqr
   - permute
   - pow
   - prod
   - qr
+  - rad2deg
   - ravel
   - real
   - reciprocal
   - remainder
   - renorm
-  - rad2deg
-  - tile
   - repeat_interleave
   - reshape
   - reshape
-  - round
+  - resize
   - rot90
+  - round
   - rsqrt
-  - sum_to_size
   - scatter
   - sgn
   - short
@@ -745,7 +753,8 @@ tensor:
   - sub
   - sub_
   - subtract
-  - subtract
+  - sum
+  - sum_to_size
   - svd
   - swapaxes
   - swapdims
@@ -753,13 +762,13 @@ tensor:
   - take
   - tan
   - tanh
-  - trace
-  - swapaxes
+  - tensor_split
   - tile
+  - to
   - topk
-  - tril
-  - tensor_split
+  - trace
   - transpose
+  - tril
   - true_divide
   - trunc
   - unbind
@@ -769,17 +778,6 @@ tensor:
   - view
   - where
   - xlogy
-  - from_numpy
-  - std
-  - take
-  - var
-  - all
-  - any
-  - copy
-  - diagonal
-  - flatten
-  - resize
-  - sum
 
 mint.ops:
   - abs

msprobe/mindspore/dump/jit_dump.py

@@ -16,6 +16,7 @@
 import os
 from collections import defaultdict
 
+import mindspore
 from mindspore._c_expression import PyNativeExecutor_
 try:
     from mindspore.common.api import _MindsporeFunctionExecutor
@@ -25,7 +26,10 @@ except ImportError:
 from msprobe.core.common.log import logger
 from msprobe.core.common.const import Const
 from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs
-from msprobe.mindspore.dump.hook_cell.api_registry import api_register
+from msprobe.mindspore.dump.hook_cell.api_register import get_api_register
+
+
+_api_register = get_api_register()
 
 
 def dump_jit(name, in_feat, out_feat, is_forward):
@@ -57,7 +61,7 @@ def dump_jit(name, in_feat, out_feat, is_forward):
 class JitDump(_MindsporeFunctionExecutor):
     dump_config = None
     jit_enable = False
-    jit_dump_switch = True
+    jit_dump_switch = False
     jit_count = defaultdict(int)
 
     def __init__(self, *args, **kwargs):
@@ -68,8 +72,7 @@ class JitDump(_MindsporeFunctionExecutor):
         self._executor = PyNativeExecutor_.get_instance()
 
     def __call__(self, *args, **kwargs):
-        if JitDump.jit_dump_switch:
-            api_register.api_set_ori_func()
+        _api_register.restore_all_api()
         out = super().__call__(*args, **kwargs)
         if JitDump.jit_dump_switch and len(args) > 0:
             if self.name and self.name != "construct":
@@ -79,8 +82,7 @@ class JitDump(_MindsporeFunctionExecutor):
             JitDump.jit_enable = True
         elif len(args) == 0:
             logger.warning(f"The jit function {self.name} has no input arguments, nothing will be dumped.")
-        if JitDump.jit_dump_switch:
-            api_register.api_set_hook_func()
+        _api_register.register_all_api()
         return out
 
     @classmethod
@@ -101,9 +103,12 @@ class JitDump(_MindsporeFunctionExecutor):
 
     def grad(self, obj, grad, weights, grad_position, *args, **kwargs):
         if JitDump.jit_dump_switch and JitDump.jit_enable:
-            api_register.api_set_ori_func()
-        output = self._executor.grad(grad, obj, weights, grad_position, *args, *(kwargs.values()))
+            _api_register.restore_all_api()
+        if mindspore.__version__ >= "2.5":
+            output = self._executor.grad(grad, obj, weights, grad_position, False, *args, *(kwargs.values()))
+        else:
+            output = self._executor.grad(grad, obj, weights, grad_position, *args, *(kwargs.values()))
         if JitDump.jit_dump_switch and JitDump.jit_enable:
             dump_jit(obj, args, None, False)
-            api_register.api_set_hook_func()
+            _api_register.register_all_api()
         return output

msprobe/mindspore/dym_loader/hook_dynamic_loader.cc

@@ -18,37 +18,10 @@
 #include <sys/stat.h>
 #include <cstdlib>
 #include <cstring>
+#include <pybind11/embed.h>
 #include "utils/log_adapter.h"
 
-namespace {
-
-// Utility function to check if a file path is valid
-bool IsValidPath(const std::string &path) {
-  struct stat fileStat;
-  if (stat(path.c_str(), &fileStat) != 0) {
-    MS_LOG(ERROR) << "File does not exist or cannot be accessed: " << path;
-    return false;
-  }
-
-  if (S_ISLNK(fileStat.st_mode)) {
-    MS_LOG(ERROR) << "File is a symbolic link, which is not allowed: " << path;
-    return false;
-  }
-
-  if (!S_ISREG(fileStat.st_mode)) {
-    MS_LOG(ERROR) << "File is not a regular file: " << path;
-    return false;
-  }
-
-  if (path.substr(path.find_last_of(".")) != ".so") {
-    MS_LOG(ERROR) << "File is not a .so file: " << path;
-    return false;
-  }
-
-  return true;
-}
-
-}  // namespace
+namespace py = pybind11;
 
 HookDynamicLoader &HookDynamicLoader::GetInstance() {
   static HookDynamicLoader instance;
@@ -65,38 +38,31 @@ bool HookDynamicLoader::loadFunction(void *handle, const std::string &functionNa
   return true;
 }
 
-bool HookDynamicLoader::validateLibraryPath(const std::string &libPath) {
-  char *realPath = realpath(libPath.c_str(), nullptr);
-  if (!realPath) {
-    MS_LOG(WARNING) << "Failed to resolve realpath for the library: " << libPath;
-    return false;
-  }
-
-  bool isValid = IsValidPath(realPath);
-  free(realPath);  // Free memory allocated by realpath
-  return isValid;
-}
-
 bool HookDynamicLoader::LoadLibrary() {
-  const char *libPath = std::getenv("HOOK_TOOL_PATH");
-  if (!libPath) {
-    MS_LOG(WARNING) << "HOOK_TOOL_PATH is not set!";
-    return false;
-  }
-
-  std::string resolvedLibPath(libPath);
-  if (!validateLibraryPath(resolvedLibPath)) {
-    MS_LOG(WARNING) << "Library path validation failed.";
-    return false;
-  }
-
+  std::string msprobePath = "";
+  // 获取gil锁
+  py::gil_scoped_acquire acquire;
+  try {
+    py::module msprobeMod = py::module::import("msprobe.lib._msprobe_c");
+		if (!py::hasattr(msprobeMod, "__file__")) {
+			MS_LOG(WARNING) << "Adump mod not found";
+			return false;
+		}
+		msprobePath = msprobeMod.attr("__file__").cast<std::string>();
+  } catch (const std::exception& e) {
+		MS_LOG(WARNING) << "Adump mod path unable to get: " << e.what();
+		return false;
+	}
   std::lock_guard<std::mutex> lock(mutex_);
   if (handle_) {
     MS_LOG(WARNING) << "Hook library already loaded!";
     return false;
   }
-
-  handle_ = dlopen(resolvedLibPath.c_str(), RTLD_LAZY | RTLD_LOCAL);
+	if (msprobePath == "") {
+		MS_LOG(WARNING) << "Adump path not loaded";
+		return false;
+	}
+  handle_ = dlopen(msprobePath.c_str(), RTLD_LAZY | RTLD_LOCAL);
   if (!handle_) {
     MS_LOG(WARNING) << "Failed to load Hook library: " << dlerror();
     return false;
@@ -104,7 +70,7 @@ bool HookDynamicLoader::LoadLibrary() {
 
   for (const auto &functionName : functionList_) {
     if (!loadFunction(handle_, functionName)) {
-      MS_LOG(WARNING) << "Failed to load function: " << functionName;
+      MS_LOG(WARNING) << "Failed to load adump function";
       dlclose(handle_);
       handle_ = nullptr;
       return false;

msprobe/mindspore/dym_loader/hook_dynamic_loader.h

@@ -40,7 +40,6 @@ class HookDynamicLoader {
  private:
   // Helper functions
   bool loadFunction(void *handle, const std::string &functionName);
-  bool validateLibraryPath(const std::string &libPath);
 
   HookDynamicLoader() = default;
 

msprobe/mindspore/free_benchmark/api_pynative_self_check.py

@@ -19,6 +19,7 @@ import os
 import traceback
 
 import mindspore as ms
+
 from msprobe.core.common.const import Const
 from msprobe.core.common.exceptions import DistributedNotInitializedError
 from msprobe.core.common.file_utils import check_path_length, load_yaml
@@ -27,7 +28,7 @@ from msprobe.mindspore.common.const import FreeBenchmarkConst
 from msprobe.mindspore.common.log import logger
 from msprobe.mindspore.common.utils import get_rank_if_initialized
 from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
-from msprobe.mindspore.dump.hook_cell.api_registry import api_register
+from msprobe.mindspore.dump.hook_cell.api_register import get_api_register
 from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell
 from msprobe.mindspore.free_benchmark.common.config import Config
 from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams
@@ -37,6 +38,9 @@ from msprobe.mindspore.free_benchmark.perturbation.perturbation_factory import P
 from msprobe.mindspore.runtime import Runtime
 
 
+_api_register = get_api_register()
+
+
 class ApiPyNativeSelfCheck:
     def __init__(self, config: DebuggerConfig):
         Config.is_enable = True
@@ -60,8 +64,8 @@ class ApiPyNativeSelfCheck:
         self.store_original_func()
 
     def handle(self):
-        api_register.initialize_hook(self.build_hook)
-        api_register.api_set_hook_func()
+        _api_register.initialize_hook(self.build_hook)
+        _api_register.register_all_api()
 
     def build_hook(self, api_name):
         def pre_hook(cell, input_data):
@@ -166,13 +170,13 @@ def check_self(api_name_with_id, output, ori_func, *args, **kwargs):
         return ret
 
     logger.info(f"[{api_name_with_id}] is {Config.handler_type}ing.")
-    api_register.api_set_ori_func()
+    _api_register.restore_all_api()
 
     try:
         perturbation = PerturbationFactory.create(api_name_with_id)
         params.fuzzed_result = perturbation.handle(params)
         if params.fuzzed_result is False:
-            api_register.api_set_hook_func()
+            _api_register.register_all_api()
             return ret
         if Config.stage == Const.BACKWARD:
             params.original_result = Tools.get_grad(params.original_func, *params.args, **params.kwargs)
@@ -183,7 +187,7 @@ def check_self(api_name_with_id, output, ori_func, *args, **kwargs):
         logger.error(f"[{api_name_with_id}] Error: {str(e)}")
         logger.error(f"[{api_name_with_id}] Error detail: {traceback.format_exc()}")
 
-    api_register.api_set_hook_func()
+    _api_register.register_all_api()
     return ret
 
 

msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from msprobe.mindspore.common.const import FreeBenchmarkConst
+from msprobe.mindspore.common.log import logger
 from msprobe.mindspore.free_benchmark.common.config import Config
 from msprobe.mindspore.free_benchmark.perturbation.add_noise import AddNoisePerturbation
 from msprobe.mindspore.free_benchmark.perturbation.bit_noise import BitNoisePerturbation
@@ -41,4 +42,5 @@ class PerturbationFactory:
         if perturbation:
             return perturbation(api_name_with_id)
         else:
-            raise Exception(f'{Config.pert_type} is a invalid perturbation type')
+            logger.error(f'{Config.pert_type} is a invalid perturbation type')
+            raise ValueError

msprobe/mindspore/free_benchmark/self_check_tool_factory.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd.
 # All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0  (the "License");
@@ -14,6 +14,7 @@
 # limitations under the License.
 
 from msprobe.mindspore.common.const import Const
+from msprobe.core.common.log import logger
 from msprobe.mindspore.debugger.debugger_config import DebuggerConfig
 from msprobe.mindspore.free_benchmark.api_pynative_self_check import ApiPyNativeSelfCheck
 
@@ -41,8 +42,10 @@ class SelfCheckToolFactory:
     def create(config: DebuggerConfig):
         tool = SelfCheckToolFactory.tools.get(config.level)
         if not tool:
-            raise Exception(f"{config.level} is not supported.")
+            logger.error(f"{config.level} is not supported.")
+            raise ValueError
         tool = tool.get(config.execution_mode)
         if not tool:
-            raise Exception(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.")
+            logger.error(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.")
+            raise ValueError
         return tool(config)

msprobe/mindspore/grad_probe/global_context.py

@@ -16,6 +16,7 @@
 import os
 import threading
 from typing import Dict, Union, Tuple
+import time
 
 from msprobe.core.common.utils import is_int
 from msprobe.core.common.file_utils import create_directory, check_path_before_create
@@ -68,6 +69,7 @@ class GlobalContext:
             create_directory(self._setting.get(GradConst.OUTPUT_PATH))
         else:
             logger.warning("The output_path exists, the data will be covered.")
+        self._setting[GradConst.TIME_STAMP] = str(int(time.time()))
 
     def get_context(self, key: str):
         if key not in self._setting:

msprobe/mindspore/grad_probe/grad_analyzer.py

@@ -111,7 +111,8 @@ class CSVGenerator(Process):
         output_path = context.get_context(GradConst.OUTPUT_PATH)
         self.level = context.get_context(GradConst.LEVEL)
         self.bounds = context.get_context(GradConst.BOUNDS)
-        self.dump_dir = f"{output_path}/rank{rank_id}/Dump/"
+        time_stamp = context.get_context(GradConst.TIME_STAMP)
+        self.dump_dir = f"{output_path}/rank{rank_id}/Dump{time_stamp}/"
         self.save_dir = f"{output_path}/rank{rank_id}/"
         self.current_step = None
         self.stop_event = multiprocessing.Event()

msprobe/mindspore/grad_probe/hook.py

@@ -49,12 +49,10 @@ class HookInput:
         self.param_list = grad_context.get_context(GradConst.PARAM_LIST)
         self.rank_id = get_rank_id()
         output_path = grad_context.get_context(GradConst.OUTPUT_PATH)
-        self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", "Dump")
+        time_stamp = grad_context.get_context(GradConst.TIME_STAMP)
+        self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", f"Dump{time_stamp}")
         self.save_dir = os.path.join(output_path, f"rank{self.rank_id}")
         self.step_finish_flag = os.path.join(self.dump_dir, GradConst.STEP_FINISH)
-        if os.path.exists(self.save_dir):
-            logger.warning(f"Delete existing path {self.save_dir}.")
-            remove_path(self.save_dir)
         self.level = grad_context.get_context(GradConst.LEVEL)
         self.bounds = grad_context.get_context(GradConst.BOUNDS)
         self.mode = mindspore.get_context("mode")

msprobe/mindspore/monitor/distributed/wrap_distributed.py

@@ -281,7 +281,7 @@ def create_hooks(context, monitor):
     global RANK
     pre_hooks = []
     hooks = []
-    RANK = str(get_rank())
+    RANK = get_rank()
     if communication.GlobalComm.INITED and RANK not in monitor.module_rank_list and monitor.module_rank_list != []:
         return [pre_hooks, hooks]