mindstudio-probe 8.1.1__py3-none-any.whl → 8.1.2__py3-none-any.whl

msprobe/mindspore/monitor/utils.py

@@ -12,16 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-import os
-import re
-from datetime import datetime
 from mindspore import dtype as mstype, Tensor
 
 from msprobe.mindspore.monitor.features import FUNC_MAP
-from msprobe.core.common.const import MonitorConst
-from msprobe.core.common.utils import is_int
-from msprobe.core.common.log import logger
-from msprobe.core.common.file_utils import check_file_or_directory_path
 
 
 def get_single_metrics(op_list, tag, tensor, eps=1e-8, output=None):
@@ -82,248 +75,3 @@ def is_skip_step(step, start_step, step_interval, has_collect_times=0, collect_t
     :return: whether skip or not, bool
     """
     return step < start_step or (step - start_step) % step_interval != 0 or has_collect_times >= collect_times
-
-
-def validate_ops(ops):
-    if not isinstance(ops, list):
-        raise TypeError("ops should be a list")
-    valid_ops = []
-    for op in ops:
-        if op not in MonitorConst.OP_LIST:
-            logger.warning(f"op {op} is not supported. Optional ops: {MonitorConst.OP_LIST}")
-            continue
-        valid_ops.append(op)
-    if not valid_ops:
-        default_op = MonitorConst.OP_LIST[0]
-        valid_ops.append(default_op)
-        logger.info(f"There is no valid ops, default op {default_op} is used")
-    # 增加默认shape和dtype参数
-    if "shape" not in valid_ops:
-        valid_ops.append("shape")
-    if "dtype" not in valid_ops:
-        valid_ops.append("dtype")
-    return valid_ops
-
-
-def validate_ranks(ranks):
-    if not isinstance(ranks, list):
-        raise TypeError("module_ranks should be a list")
-    for rank in ranks:
-        if not isinstance(rank, int):
-            raise TypeError(f"element in module_ranks should be a int, get {type(rank)}")
-
-
-def validate_targets(targets):
-    if not isinstance(targets, dict):
-        raise TypeError('targets in config.json should be a dict')
-    for module_name, field in targets.items():
-        if not isinstance(module_name, str):
-            raise TypeError('key of targets should be module_name[str] in config.json')
-        if not isinstance(field, dict):
-            raise TypeError('values of targets should be cared filed e.g. {"input": "tensor"} in config.json')
-
-
-def validate_print_struct(print_struct):
-    if not isinstance(print_struct, bool):
-        raise TypeError("print_struct should be a bool")
-
-
-def validate_ur_distribution(ur_distribution):
-    if not isinstance(ur_distribution, bool):
-        raise TypeError('ur_distribution should be a bool')
-
-
-def validate_xy_distribution(xy_distribution):
-    if not isinstance(xy_distribution, bool):
-        raise TypeError('xy_distribution should be a bool')
-
-
-def validate_wg_distribution(wg_distribution):
-    if not isinstance(wg_distribution, bool):
-        raise TypeError('wg_distribution should be a bool')
-
-
-def validate_mg_distribution(mg_distribution):
-    if not isinstance(mg_distribution, bool):
-        raise TypeError('mg_distribution should be a bool')
-
-
-def validate_param_distribution(param_distribution):
-    if not isinstance(param_distribution, bool):
-        raise TypeError('param_distribution should be a bool')
-
-
-def validate_cc_distribution(cc_distribution):
-    if not isinstance(cc_distribution, dict):
-        raise TypeError('cc_distribution should be a dictionary')
-    expected_keys = {
-        'enable': bool,
-        'cc_codeline': list,
-        'cc_pre_hook': bool,
-        'cc_log_only': bool
-    }
-    for key, value in cc_distribution.items():
-        if key in expected_keys:
-            if not isinstance(value, expected_keys[key]):
-                raise TypeError(f'cc_distribution {key} should be a {expected_keys[key].__name__}')
-        else:
-            raise TypeError(f'{key} of cc_distribution is not supported.')
-
-
-def validate_alert(alert):
-    if not isinstance(alert, dict):
-        raise TypeError('alert should be a dictionary')
-    rules = alert.get('rules')
-    if rules and isinstance(rules, list):
-        for rule in rules:
-            rule_name = rule.get("rule_name")
-            if rule_name and rule_name not in MonitorConst.RULE_NAME:
-                raise TypeError(f"{rule_name} is not supported")
-            args = rule.get("args")
-            if args and isinstance(args, dict):
-                threshold = args.get("threshold")
-                if not isinstance(threshold, (float, int)) or threshold < 0:
-                    raise TypeError('threshold must be float and not less than 0')
-    dump = alert.get('dump')
-    if dump and not isinstance(dump, bool):
-        raise TypeError('dump must be bool.')
-
-
-def validate_step_count_per_record(step_count_per_record):
-    if not is_int(step_count_per_record):
-        raise TypeError('step_count_per_record must be int.')
-    if step_count_per_record < 1:
-        raise ValueError("step_count_per_record must greater than 0")
-    if step_count_per_record > 1e6:
-        raise ValueError("step_count_per_record must smaller than 1e6")
-
-
-def validate_start_step(start_step):
-    if not is_int(start_step):
-        raise TypeError('start_step must be int.')
-    if start_step < 0:
-        raise ValueError("start_step must greater than 0")
-    if start_step > 1e8:
-        raise ValueError("start_step must smaller than 1e8")
-
-
-def validate_step_interval(step_interval):
-    if not is_int(step_interval):
-        raise TypeError('step_interval must be int.')
-    if step_interval < 1:
-        raise ValueError("step_interval must greater than 1")
-    if step_interval > 1e8:
-        raise ValueError("step_interval must smaller than 1e8")
-
-
-def validate_collect_times(collect_times):
-    if not is_int(collect_times):
-        raise TypeError('collect_times must be int.')
-    if collect_times < 1:
-        raise ValueError("collect_times must greater than 1")
-
-
-def validate_dynamic_on(dynamic_on):
-    if not isinstance(dynamic_on, bool):
-        raise TypeError('dynamic_on should be a bool')
-
-
-def validate_monitor_mbs_grad(monitor_mbs_grad):
-    if not isinstance(monitor_mbs_grad, bool):
-        logger.warning(f'monitor_mbs_grad should be a bool, actual value is {monitor_mbs_grad}.')
-        return False
-    return monitor_mbs_grad
-
-
-def validate_config(config):
-    config['ops'] = validate_ops(config.get('ops', []))
-
-    eps = config.get('eps', 1e-8)
-    if not isinstance(eps, float):
-        raise TypeError("eps should be a float")
-
-    ranks = config.get("module_ranks", [])
-    validate_ranks(ranks)
-
-    targets = config.get("targets", {})
-    validate_targets(targets)
-
-    print_struct = config.get('print_struct', False)
-    validate_print_struct(print_struct)
-
-    ur_distribution = config.get('ur_distribution', False)
-    validate_ur_distribution(ur_distribution)
-
-    xy_distribution = config.get('xy_distribution', False)
-    validate_xy_distribution(xy_distribution)
-
-    wg_distribution = config.get('wg_distribution', False)
-    validate_wg_distribution(wg_distribution)
-
-    mg_distribution = config.get('mg_distribution', False)
-    validate_mg_distribution(mg_distribution)
-
-    param_distribution = config.get('param_distribution', False)
-    validate_param_distribution(param_distribution)
-
-    cc_distribution = config.get('cc_distribution', {})
-    validate_cc_distribution(cc_distribution)
-
-    alert = config.get('alert', {})
-    validate_alert(alert)
-
-    step_count_per_record = config.get('step_count_per_record', 1)
-    validate_step_count_per_record(step_count_per_record)
-
-    start_step = config.get('start_step', 0)
-    validate_start_step(start_step)
-
-    step_interval = config.get('step_interval', 1)
-    validate_step_interval(step_interval)
-
-    collect_times = config.get('collect_times', int(1e8))
-    validate_collect_times(collect_times)
-
-    config["monitor_mbs_grad"] = validate_monitor_mbs_grad(config.get('monitor_mbs_grad', False))
-
-    dynamic_on = config.get('dynamic_on', False)
-    validate_dynamic_on(dynamic_on)
-
-    if not targets:
-        if xy_distribution:
-            config["all_xy"] = True
-        config["targets"] = {"": {}}
-        config["is_select"] = False
-    else:
-        config["is_select"] = True
-
-
-def time_str2time_digit(time_str):
-    time_format = '%b%d_%H-%M-%S'
-    try:
-        time_digit = datetime.strptime(time_str, time_format)
-    except Exception as e:
-        raise RuntimeError(f"illegal timestamp: {time_str}, timestamp should be prefix \
-                           of existing output dirpath, like 'Dec03_21-34-40'.") from e
-    return time_digit
-
-
-def get_target_output_dir(monitor_path, time_start, time_end):
-    check_file_or_directory_path(monitor_path, isdir=True)
-    time_start = time_str2time_digit(time_start) if time_start is not None else time_start
-    time_end = time_str2time_digit(time_end) if time_end is not None else time_end
-    if time_start and time_end and time_start > time_end:
-        raise ValueError(f"time_start({time_start}) greater than time_end({time_end})")
-    result = {}
-    for dirname in os.listdir(monitor_path):
-        match = re.match(MonitorConst.OUTPUT_DIR_PATTERN, dirname)
-        if not match:
-            continue
-        time_tag = match.group(1)
-        rank = match.group(2)
-        target_time = time_str2time_digit(time_tag)
-        start_ok = time_start is None or target_time >= time_start
-        end_ok = time_end is None or target_time <= time_end
-        if start_ok and end_ok:
-            result[rank] = os.path.join(monitor_path, dirname)
-    return result

msprobe/mindspore/ms_config.py

@@ -14,7 +14,6 @@
 # limitations under the License.
 
 from msprobe.core.common.const import Const
-from msprobe.core.common.file_utils import load_json
 from msprobe.core.common.utils import is_int
 from msprobe.core.common_config import BaseConfig, CommonConfig
 from msprobe.core.grad_probe.constant import level_adp

msprobe/mindspore/overflow_check/overflow_check_tool_factory.py

@@ -48,4 +48,4 @@ class OverflowCheckToolFactory:
             logger.error(f"Overflow check is not supported in {config.execution_mode} mode "
                          f"when level is {config.level}.")
             raise ValueError
-        return tool(config)
+        return (tool(config),)

msprobe/nan_analyze/graph.py

@@ -16,6 +16,7 @@
 from dataclasses import dataclass
 from msprobe.core.common.const import Const
 from msprobe.core.common.log import logger
+from msprobe.core.common.exceptions import MsprobeException
 from msprobe.nan_analyze.utils import FileCache, RankPath, is_ignore_op, check_item_anomaly, NanAnalyseConst
 
 
@@ -52,6 +53,9 @@ class DataNode:
 
     def find_stack(self, stack_info):
         for item in stack_info.values():
+            if not isinstance(item, list):
+                raise MsprobeException(MsprobeException.UNSUPPORTED_TYPE_ERROR,
+                                       f'The value\'s type in stack.json should be a list, not {type(item)}!')
             if len(item) >= 2 and self.op_name in item[0]:
                 return item[1]
         return {}

msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py

@@ -33,7 +33,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator
 from msprobe.pytorch.common import parse_json_info_forward_backward
 from msprobe.pytorch.common.log import logger
 from msprobe.core.common.file_utils import FileChecker, check_file_suffix, check_link, FileOpen, \
-    create_directory, load_json, save_json
+    create_directory, load_json, save_json, read_csv
 from msprobe.core.common.file_utils import remove_path
 from msprobe.core.common.const import FileCheckConst, Const
 from msprobe.core.common.utils import CompareException
@@ -76,9 +76,18 @@ def split_json_file(input_file, num_splits, filter_api):
             }
         }
         split_filename = os.path.join(input_dir, f"temp_part{i}.json")
-        save_json(split_filename, temp_data)
         split_files.append(split_filename)
-
+        try:
+            save_json(split_filename, temp_data)
+        except Exception as e:
+            logger.error(f"An error occurred while saving split file: {e}")
+            for file in split_files:
+                try:
+                    remove_path(file)
+                except FileNotFoundError:
+                    logger.error(f"File not found and could not be deleted: {file}")
+            msg = 'ERROR: Split json file failed, please check the input file and try again.'
+            raise CompareException(CompareException.PARSE_FILE_ERROR, msg) from e
     return split_files, total_items
 
 
@@ -134,9 +143,9 @@ def run_parallel_ut(config):
 
     def update_progress_bar(progress_bar, result_csv_path):
         while any(process.poll() is None for process in processes):
-            with FileOpen(result_csv_path, 'r') as result_file:
-                completed_items = len(result_file.readlines()) - 1
-                progress_bar.update(completed_items - progress_bar.n)
+            result_file = read_csv(result_csv_path)
+            completed_items = len(result_file)
+            progress_bar.update(completed_items - progress_bar.n)
             time.sleep(1)
 
     for api_info in config.api_files:

msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py

@@ -293,7 +293,7 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict
     if grad_input_index is not None:
         grad_index = grad_input_index.get('grad_index')
 
-    if need_backward:
+    if need_backward and out is not None:
         if need_to_backward(grad_index, out):
             backward_args = backward_content[api_full_name].get("input")
             func_options = {

msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py

@@ -43,9 +43,9 @@ CommonCompareConfig = namedtuple('CommonCompareConfig', ['compare', 'handle_func
 
 
 def get_gpu_device():
+    is_gpu = False
     try:
         import torch_npu
-        is_gpu = False
     except ImportError:
         is_gpu = True
     return is_gpu

msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/utils.py

@@ -111,10 +111,8 @@ def load_ssl_pem(key_file, cert_file, ca_file, crl_file):
 
     try:
         # your_private_key_password
-        passphrase = ""
-        if not passphrase:
-            import pwinput
-            passphrase = pwinput.pwinput("Enter your password: ")
+        import pwinput
+        passphrase = pwinput.pwinput("Enter your password: ")
         with FileOpen(key_file, "rb") as f:
             key = crypto.load_privatekey(crypto.FILETYPE_PEM, f.read(), passphrase.encode())
             del passphrase

msprobe/pytorch/common/utils.py

@@ -264,10 +264,6 @@ class Const:
     NPU = 'NPU'
     DISTRIBUTED = 'Distributed'
 
-    HIFLOAT8_TYPE = "torch_npu.HiFloat8Tensor"
-    FLOAT8_E5M2_TYPE = "torch.float8_e5m2"
-    FLOAT8_E4M3FN_TYPE = "torch.float8_e4m3fn"
-
     RAISE_PRECISION = {
         torch.float16: torch.float32,
         torch.bfloat16: torch.float32,
@@ -483,18 +479,6 @@ def is_torch_nn_module(variable):
     return isinstance(variable, torch.nn.Module) and not isinstance(variable, torch.jit.ScriptModule)
 
 
-def is_hifloat8_tensor(tensor):
-    if not is_gpu and hasattr(torch_npu, "HiFloat8Tensor") and isinstance(tensor, torch_npu.HiFloat8Tensor):
-        return True
-    return False
-
-
-def is_float8_tensor(tensor):
-    if str(tensor.dtype) in [Const.FLOAT8_E5M2_TYPE, Const.FLOAT8_E4M3FN_TYPE]:
-        return True
-    return is_hifloat8_tensor(tensor)
-
-
 def register_forward_pre_hook(module, forward_pre_hook):
     if torch_version_above_or_equal_2:
         module.register_forward_pre_hook(forward_pre_hook, with_kwargs=True)

msprobe/pytorch/compare/pt_compare.py

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from msprobe.core.common.utils import CompareException
+from msprobe.core.common.log import logger
 from msprobe.core.compare.acc_compare import Comparator, ModeConfig, MappingConfig, setup_comparison
 from msprobe.pytorch.compare.utils import read_pt_data
 
@@ -24,6 +26,9 @@ def read_real_data(npu_dir, npu_data_name, bench_dir, bench_data_name, _) -> tup
 
 
 def compare(input_param, output_path, **kwargs):
+    if not isinstance(input_param, dict):
+        logger.error("input_param should be dict, please check!")
+        raise CompareException(CompareException.INVALID_OBJECT_TYPE_ERROR)
     config = setup_comparison(input_param, output_path, **kwargs)
 
     mode_config = ModeConfig(config.stack_mode, config.auto_analyze, config.fuzzy_match,

msprobe/pytorch/debugger/debugger_config.py

@@ -98,6 +98,11 @@ class DebuggerConfig:
 
     def check_model(self, instance, start_model, token_range=None):
         instance.model = start_model if start_model is not None else instance.model
+
+        if token_range and not instance.model:
+            error_info = "The 'model' parameter must be provided when token_range is not None"
+            raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, error_info)
+        
         if self.level not in [Const.LEVEL_L0, Const.LEVEL_MIX] and token_range is None:
             return
 
@@ -110,18 +115,20 @@ class DebuggerConfig:
         if is_torch_nn_module(instance.model):
             return
 
-        error_model = None
         if isinstance(instance.model, (list, tuple)):
+            error_model = None
             for model in instance.model:
                 if not is_torch_nn_module(model):
                     error_model = model
                     break
+            if error_model is not None:
+                error_info = (f"The 'model' parameter must be a torch.nn.Module or list[torch.nn.Module] "
+                            f"type, currently there is an unsupported {type(error_model)} type.")
+                raise MsprobeException(
+                    MsprobeException.INVALID_PARAM_ERROR, error_info)
         else:
-            error_model = instance.model
-
-        if error_model is not None:
             error_info = (f"The 'model' parameter must be a torch.nn.Module or list[torch.nn.Module] "
-                          f"type, currently there is an unsupported {type(error_model)} type.")
+                          f"type, currently there is an unsupported {type(instance.model)} type.")
             raise MsprobeException(
                 MsprobeException.INVALID_PARAM_ERROR, error_info)
 

msprobe/pytorch/debugger/precision_debugger.py

@@ -17,7 +17,7 @@ from torch.utils.data import dataloader
 
 from msprobe.core.common.const import Const, MsgConst
 from msprobe.core.common.exceptions import MsprobeException
-from msprobe.core.common.utils import check_token_range
+from msprobe.core.common.utils import check_token_range, ThreadSafe
 from msprobe.core.debugger.precision_debugger import BasePrecisionDebugger
 from msprobe.pytorch.common.log import logger
 from msprobe.pytorch.common.utils import check_save_param, is_torch_nn_module
@@ -81,6 +81,7 @@ class PrecisionDebugger(BasePrecisionDebugger):
         return func_wrapper
 
     @classmethod
+    @ThreadSafe.synchronized
     def start(cls, model=None, token_range=None):
         instance = cls._get_instance()
         if instance is None:
@@ -95,6 +96,7 @@ class PrecisionDebugger(BasePrecisionDebugger):
             instance.service.start(instance.model, token_range)
 
     @classmethod
+    @ThreadSafe.synchronized
     def stop(cls):
         instance = cls._get_instance()
         if instance is None:
@@ -105,6 +107,7 @@ class PrecisionDebugger(BasePrecisionDebugger):
             instance.service.stop()
 
     @classmethod
+    @ThreadSafe.synchronized
     def step(cls):
         instance = cls._get_instance()
         if instance is None:
@@ -112,6 +115,7 @@ class PrecisionDebugger(BasePrecisionDebugger):
         cls._instance.service.step()
 
     @classmethod
+    @ThreadSafe.synchronized
     def monitor(cls, model):
         if not cls._instance:
             raise Exception(MsgConst.NOT_CREATED_INSTANCE)
@@ -120,6 +124,7 @@ class PrecisionDebugger(BasePrecisionDebugger):
         cls._instance.gm.monitor(model)
 
     @classmethod
+    @ThreadSafe.synchronized
     def save(cls, variable, name, save_backward=True):
         instance = cls._instance
         if not instance:
@@ -143,6 +148,7 @@ class PrecisionDebugger(BasePrecisionDebugger):
             dataloader._BaseDataLoaderIter.__next__ = self._iter_tracer(dataloader._BaseDataLoaderIter.__next__)
 
 
+@ThreadSafe.synchronized
 def module_dump(module, dump_name):
     if not is_torch_nn_module(module):
         raise MsprobeException(
@@ -164,6 +170,7 @@ def module_dump(module, dump_name):
     instance.module_dumper.start_module_dump(module, dump_name)
 
 
+@ThreadSafe.synchronized
 def module_dump_end():
     instance = PrecisionDebugger._instance
     if not instance:

msprobe/pytorch/dump/module_dump/hook_wrapper.py

@@ -21,13 +21,11 @@ from torch.utils.hooks import BackwardHook
 from msprobe.core.common.const import Const
 from msprobe.core.common.decorator import recursion_depth_decorator
 from msprobe.pytorch.common.log import logger
-from msprobe.pytorch.common.utils import is_float8_tensor
 
 
 def wrap_setup_backward_hook(func):
     def requires_clone(tensor):
-        return isinstance(tensor, torch.Tensor) and not is_float8_tensor(tensor) and \
-            tensor.requires_grad and torch.is_grad_enabled()
+        return isinstance(tensor, torch.Tensor) and tensor.requires_grad and torch.is_grad_enabled()
 
     @recursion_depth_decorator("Dump: wrap_setup_backward_hook.parse_tensor", max_depth=Const.DUMP_MAX_DEPTH)
     def parse_tensor(item, tensor_list):