mindstudio-probe 8.1.1__py3-none-any.whl → 8.1.2__py3-none-any.whl

msprobe/core/debugger/precision_debugger.py

@@ -18,7 +18,7 @@ import os
 from msprobe.core.common.const import Const, FileCheckConst, MsgConst
 from msprobe.core.common.exceptions import MsprobeException
 from msprobe.core.common.file_utils import FileChecker, load_json
-from msprobe.core.common.utils import get_real_step_or_rank, check_init_step
+from msprobe.core.common.utils import get_real_step_or_rank, check_init_step, ThreadSafe
 from msprobe.core.common_config import CommonConfig
 
 
@@ -27,13 +27,14 @@ class BasePrecisionDebugger:
     tasks_not_need_debugger = [Const.GRAD_PROBE]
 
     def __new__(cls, *args, **kwargs):
-        if cls._instance is None:
-            cls._instance = super(BasePrecisionDebugger, cls).__new__(cls)
-            cls._instance.config = None
-            cls._instance.enable_dataloader = False
-            cls._instance.initialized = False
-            cls.service = None
-            cls.first_start = False
+        if not cls._instance:
+            with ThreadSafe():
+                if not cls._instance:
+                    cls._instance = super(BasePrecisionDebugger, cls).__new__(cls)
+                    cls._instance.config = None
+                    cls._instance.initialized = False
+                    cls.service = None
+                    cls.first_start = False
         return cls._instance
 
     def __init__(
@@ -83,11 +84,13 @@ class BasePrecisionDebugger:
         raise NotImplementedError("Subclass must implement _get_task_config")
 
     @classmethod
+    @ThreadSafe.synchronized
     def forward_backward_dump_end(cls):
         instance = cls._instance
         instance.stop()
 
     @classmethod
+    @ThreadSafe.synchronized
     def set_init_step(cls, step):
         instance = cls._instance
         if not instance:
@@ -97,6 +100,7 @@ class BasePrecisionDebugger:
         instance.service.loop = 0
 
     @classmethod
+    @ThreadSafe.synchronized
     def register_custom_api(cls, module, api, api_prefix=None):
         if not api_prefix:
             api_prefix = getattr(module, "__name__", "Custom")
@@ -112,6 +116,7 @@ class BasePrecisionDebugger:
         instance.service.register_custom_api(module, api, api_prefix)
 
     @classmethod
+    @ThreadSafe.synchronized
     def restore_custom_api(cls, module, api):
         if not hasattr(module, api):
             raise MsprobeException(

msprobe/core/hook_manager.py

@@ -13,12 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
-from abc import ABC, abstractmethod
 import os
+import threading
+from abc import ABC, abstractmethod
+from collections import defaultdict
 
+from msprobe.core.common.log import logger
 from msprobe.core.common.runtime import Runtime
-from msprobe.core.common.utils import Const
+from msprobe.core.common.utils import Const, ThreadSafe
 from msprobe.core.data_dump.data_processor.base import (ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs)
 
 
@@ -31,7 +33,7 @@ class HookSet:
 
 
 class BaseHookManager(ABC):
-    inner_switch = False
+    inner_switch = defaultdict(bool)
     hook_handle_dict = {}
     params_grad_info = {}
 
@@ -86,7 +88,7 @@ class BaseHookManager(ABC):
         grad_name = ori_name + Const.SEP + Const.PARAMS_GRAD
         # 首次执行前向hook时，添加params_grad_name属性，并注册参数hook
         setattr(module, 'params_grad_name', grad_name)
-         # data_mode为forward时，不注册参数hook
+        # data_mode为forward时，不注册参数hook
         if not (Const.FORWARD in self.config.data_mode and Const.BACKWARD not in self.config.data_mode):
             for param_name, param in params_dict.items():
                 if param.requires_grad:
@@ -116,7 +118,7 @@ class BaseHookManager(ABC):
                 # 记录当前模块的参数梯度信息已占位
                 BaseHookManager.params_grad_info[grad_name] = True
 
-    def _should_execute_hook(self, hook_type, module, is_forward):
+    def _should_execute_hook(self, hook_type, module, is_forward, tid):
         is_module_hook = hook_type == Const.MODULE
         if is_module_hook and not Runtime.is_running:
             return False
@@ -124,7 +126,7 @@ class BaseHookManager(ABC):
             return False
         elif not is_module_hook and not is_forward and not module.forward_data_collected:
             return False
-        if BaseHookManager.inner_switch:
+        if BaseHookManager.inner_switch[tid]:
             return False
         if not self.data_collector or self.data_collector.data_processor.is_terminated:
             return False
@@ -132,111 +134,139 @@ class BaseHookManager(ABC):
 
     def _build_grad_hook(self, module, ori_name, param_name):
         def hook_fn(grad):
-            if not self._should_execute_hook(Const.MODULE, module, False):
+            tid = threading.get_ident()
+            if not self._should_execute_hook(Const.MODULE, module, False, tid):
                 return
-            BaseHookManager.inner_switch = True
-            self.data_collector.params_data_collect(ori_name, param_name, self._pid, grad)
-            BaseHookManager.inner_switch = False
+            with ThreadSafe():
+                BaseHookManager.inner_switch[tid] = True
+                self.data_collector.params_data_collect(ori_name, param_name, self._pid, grad)
+                BaseHookManager.inner_switch[tid] = False
             return
+
         return hook_fn
 
     def _build_forward_pre_hook(self, hook_type, full_name, api_name):
         def forward_pre_hook(module, args, kwargs=None):
+            """
+            为确保多线程场景下 L1 级别数据采集的正确性，每个封装后的 API 的 init 方法和 forward_pre_hook 需要确保在一个线程内连续完成，
+            因此在 API 的 init 方法执行 ThreadSafe.acquire() 加锁操作，
+            并且在 API 的 forward_pre_hook 方法执行 ThreadSafe.release() 释放锁操作。
+            """
             if hook_type == Const.MODULE:
                 return
-            if not self._should_execute_hook(hook_type, module, True):
+
+            tid = threading.get_ident()
+            if not self._should_execute_hook(hook_type, module, True, tid):
+                ThreadSafe.release()
                 return
+
+            module.forward_data_collected = True
+            self._add_count(api_name)
+            if getattr(self.config, "online_run_ut", False):
+                ThreadSafe.release()
+                return
+
+            BaseHookManager.inner_switch[tid] = True
             if kwargs is None:
                 kwargs = module.msprobe_input_kwargs if hasattr(module, 'msprobe_input_kwargs') else {}
-            with self._no_grad_context():
-                BaseHookManager.inner_switch = False
-                module.forward_data_collected = True
-                self._add_count(api_name)
-                module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=None)
-                self.data_collector.update_api_or_module_name(full_name)
-                if getattr(self.config, "online_run_ut", False):
-                    BaseHookManager.inner_switch = False
-                    return
-                self.data_collector.forward_input_data_collect(
-                    full_name,
-                    module,
-                    self._pid,
-                    module_input_output,
-                    self._is_recompute
-                )
-                BaseHookManager.inner_switch = False
-        return forward_pre_hook
-
-    def _build_forward_hook(self, hook_type, full_name):
-        def forward_hook(module, args, kwargs_or_output, output_or_kwargs=None):
-            if not self._should_execute_hook(hook_type, module, True):
-                self._clear_input_kwargs(module)
-                return None
-            kwargs, output = self._process_kwargs_and_output(module, hook_type, kwargs_or_output, output_or_kwargs)
-            BaseHookManager.inner_switch = True
-            self.data_collector.update_api_or_module_name(full_name)
-            module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=output)
-            with self._no_grad_context():
-                if getattr(self.config, "online_run_ut", False):
-                    if self.data_collector.scope and not self.data_collector.scope.check(full_name):
-                        return None
-                    if self.attl_manager:
-                        self.attl_manager.attl_send(full_name, args, kwargs, output)
-                    BaseHookManager.inner_switch = False
-                    return None
-                if hook_type == Const.MODULE:
-                    params_dict = self._get_params_dict(module)
-                    setattr(module_input_output, Const.PARAMS, params_dict)
-                    if params_dict:
-                        self._register_param_hook(full_name, module, params_dict)
+            try:
+                with self._no_grad_context():
+                    module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=None)
                     self.data_collector.update_api_or_module_name(full_name)
-                    self.data_collector.forward_data_collect(
-                        full_name,
-                        module,
-                        self._pid,
-                        module_input_output,
-                        self._is_recompute
-                    )
-                    self._init_params_grad_info(module, params_dict)
-                else:
-                    self.data_collector.forward_output_data_collect(
+                    self.data_collector.forward_input_data_collect(
                         full_name,
                         module,
                         self._pid,
                         module_input_output,
                         self._is_recompute
                     )
+            except Exception as e:
+                logger.error(f"The forward pre hook execution of the {full_name} API failed.")
+                raise e
+            finally:
+                BaseHookManager.inner_switch[tid] = False
+                ThreadSafe.release()
+
+        return forward_pre_hook
+
+    def _build_forward_hook(self, hook_type, full_name):
+        def forward_hook(module, args, kwargs_or_output, output_or_kwargs=None):
+            tid = threading.get_ident()
+            if not self._should_execute_hook(hook_type, module, True, tid):
                 self._clear_input_kwargs(module)
+                return None
 
-                if self.data_collector.if_return_forward_new_output():
-                    forward_new_output = self.data_collector.get_forward_new_output()
-                    BaseHookManager.inner_switch = False
-                    return forward_new_output
+            with ThreadSafe():
+                kwargs, output = self._process_kwargs_and_output(module, hook_type, kwargs_or_output, output_or_kwargs)
+                BaseHookManager.inner_switch[tid] = True
+                self.data_collector.update_api_or_module_name(full_name)
+                module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=output)
+                with self._no_grad_context():
+                    if getattr(self.config, "online_run_ut", False):
+                        if self.data_collector.scope and not self.data_collector.scope.check(full_name):
+                            return None
+                        if self.attl_manager:
+                            self.attl_manager.attl_send(full_name, args, kwargs, output)
+                        BaseHookManager.inner_switch[tid] = False
+                        return None
+                    if hook_type == Const.MODULE:
+                        params_dict = self._get_params_dict(module)
+                        setattr(module_input_output, Const.PARAMS, params_dict)
+                        if params_dict:
+                            self._register_param_hook(full_name, module, params_dict)
+                        self.data_collector.update_api_or_module_name(full_name)
+                        self.data_collector.forward_data_collect(
+                            full_name,
+                            module,
+                            self._pid,
+                            module_input_output,
+                            self._is_recompute
+                        )
+                        self._init_params_grad_info(module, params_dict)
+                    else:
+                        self.data_collector.forward_output_data_collect(
+                            full_name,
+                            module,
+                            self._pid,
+                            module_input_output,
+                            self._is_recompute
+                        )
+                    self._clear_input_kwargs(module)
+
+                    if self.data_collector.if_return_forward_new_output():
+                        forward_new_output = self.data_collector.get_forward_new_output()
+                        BaseHookManager.inner_switch[tid] = False
+                        return forward_new_output
+
+                    BaseHookManager.inner_switch[tid] = False
+                    return output
 
-                BaseHookManager.inner_switch = False
-                return output
         return forward_hook
 
     def _build_backward_hook(self, hook_type, full_name):
         def backward_hook(module, grad_input, grad_output):
-            if not self._should_execute_hook(hook_type, module, False):
-                return
-            BaseHookManager.inner_switch = True
-            self.data_collector.update_api_or_module_name(full_name)
-            if getattr(self.config, "online_run_ut", False):
-                BaseHookManager.inner_switch = False
+            tid = threading.get_ident()
+            if not self._should_execute_hook(hook_type, module, False, tid):
                 return
-            need_exchange = self._need_exchange(module) if hook_type == Const.MODULE else True
-            if need_exchange:
-                module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input)
-            else:
-                module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_input, grad_output=grad_output)
-            self.data_collector.backward_data_collect(
+
+            with ThreadSafe():
+                BaseHookManager.inner_switch[tid] = True
+                self.data_collector.update_api_or_module_name(full_name)
+                if getattr(self.config, "online_run_ut", False):
+                    BaseHookManager.inner_switch[tid] = False
+                    return
+                need_exchange = self._need_exchange(module) if hook_type == Const.MODULE else True
+                if need_exchange:
+                    module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input)
+                else:
+                    module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_input, grad_output=grad_output)
+                self.data_collector.backward_data_collect(
                     full_name,
                     module,
                     self._pid,
                     module_input_output,
                     self._is_recompute
                 )
-            BaseHookManager.inner_switch = False
+                BaseHookManager.inner_switch[tid] = False
+
         return backward_hook

msprobe/core/monitor/utils.py

@@ -0,0 +1,338 @@
+# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
+# All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0  (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from collections import namedtuple
+from datetime import timezone, timedelta
+from functools import wraps
+from datetime import datetime
+import os
+import re
+
+from msprobe.core.common.const import MonitorConst
+from msprobe.core.common.log import logger
+from msprobe.core.common.utils import is_int
+from msprobe.core.common.file_utils import check_file_or_directory_path, recursive_chmod
+
+
+beijing_tz = timezone(timedelta(hours=8))
+MVResult = namedtuple('MVResult', ("exp_avg", "exp_avg_sq", "update", "ratio"))
+
+
+class MsgConst:
+    """
+    Class for log messages const
+    """
+    SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"]
+
+
+def get_output_base_dir():
+    return os.getenv(MonitorConst.MONITOR_OUTPUT_DIR, MonitorConst.DEFAULT_MONITOR_OUTPUT_DIR)
+
+
+def filter_special_chars(func):
+    @wraps(func)
+    def func_level(msg):
+        for char in MsgConst.SPECIAL_CHAR:
+            msg = msg.replace(char, '_')
+        return func(msg)
+
+    return func_level
+
+
+def validate_ops(ops):
+    if not isinstance(ops, list):
+        raise TypeError("ops should be a list")
+    valid_ops = []
+    for op in ops:
+        if op not in MonitorConst.OP_LIST:
+            logger.warning(f"op {op} is not supported. Optional ops: {MonitorConst.OP_LIST}")
+            continue
+        valid_ops.append(op)
+    if not valid_ops:
+        default_op = MonitorConst.OP_LIST[0]
+        valid_ops.append(default_op)
+        logger.info_on_rank_0(f"There is no valid ops, default op {default_op} is used")
+    # 增加默认shape和dtype参数
+    if "shape" not in valid_ops:
+        valid_ops.append("shape")
+    if "dtype" not in valid_ops:
+        valid_ops.append("dtype")
+    return valid_ops
+
+
+def validate_ndigits(ndigits):
+    if not ndigits:
+        return
+    if not is_int(ndigits) or ndigits <= 0:
+        raise ValueError(f"ndigits({ndigits}) is not a positive integer, current is: {ndigits}.")
+    if ndigits > MonitorConst.MAX_NDIGITS:
+        raise ValueError(f"The maximum supported ndigits is {MonitorConst.MAX_NDIGITS}, current value: {ndigits}.")
+
+
+def validate_ranks(ranks):
+    if not isinstance(ranks, list):
+        raise TypeError("module_ranks should be a list")
+    for rank in ranks:
+        if not isinstance(rank, int) or isinstance(rank, bool):
+            raise TypeError(f"element in module_ranks should be a int, get {type(rank)}")
+
+
+def validate_targets(targets):
+    if not isinstance(targets, dict):
+        raise TypeError('targets in config.json should be a dict')
+    for module_name, field in targets.items():
+        if not isinstance(module_name, str):
+            raise TypeError('key of targets should be module_name[str] in config.json')
+        if not isinstance(field, dict):
+            raise TypeError('values of targets should be cared filed e.g. {"input": "tensor"} in config.json')
+
+
+def validate_print_struct(print_struct):
+    if not isinstance(print_struct, bool):
+        raise TypeError("print_struct should be a bool")
+
+
+def validate_ur_distribution(ur_distribution):
+    if not isinstance(ur_distribution, bool):
+        raise TypeError('ur_distribution should be a bool')
+
+
+def validate_xy_distribution(xy_distribution):
+    if not isinstance(xy_distribution, bool):
+        raise TypeError('xy_distribution should be a bool')
+
+
+def validate_wg_distribution(wg_distribution):
+    if not isinstance(wg_distribution, bool):
+        raise TypeError('wg_distribution should be a bool')
+
+
+def validate_mg_distribution(mg_distribution):
+    if not isinstance(mg_distribution, bool):
+        raise TypeError('mg_distribution should be a bool')
+
+
+def validate_param_distribution(param_distribution):
+    if not isinstance(param_distribution, bool):
+        raise TypeError('param_distribution should be a bool')
+
+
+def validate_cc_distribution(cc_distribution):
+    if not isinstance(cc_distribution, dict):
+        raise TypeError('cc_distribution should be a dictionary')
+    for key, value in cc_distribution.items():
+        if key == 'enable':
+            if not isinstance(value, bool):
+                raise TypeError('cc_distribution enable should be a bool')
+        elif key == 'cc_codeline':
+            if not isinstance(value, list):
+                raise TypeError('cc_distribution cc_codeline should be a list')
+        elif key == 'cc_pre_hook':
+            if not isinstance(value, bool):
+                raise TypeError('cc_distribution cc_pre_hook should be a bool')
+        elif key == 'cc_log_only':
+            if not isinstance(value, bool):
+                raise TypeError('cc_distribution cc_log_only should be a bool')
+        else:
+            raise TypeError(f'{key} of cc_distribution is not supported.')
+
+
+def validate_squash_name(squash_name):
+    if not isinstance(squash_name, bool):
+        raise TypeError('squash_name should be a bool')
+
+
+def validate_alert(alert):
+    if not isinstance(alert, dict):
+        raise TypeError('alert should be a dictionary')
+    rules = alert.get('rules')
+    if rules and isinstance(rules, list):
+        for rule in rules:
+            rule_name = rule.get("rule_name")
+            if rule_name and rule_name not in MonitorConst.RULE_NAME:
+                raise TypeError(f"{rule_name} is not supported")
+            args = rule.get("args")
+            if args and isinstance(args, dict):
+                threshold = args.get("threshold")
+                if not isinstance(threshold, (float, int)) or threshold < 0:
+                    raise TypeError('threshold must be float and not less than 0')
+    dump = alert.get('dump')
+    if dump and not isinstance(dump, bool):
+        raise TypeError('dump must be bool.')
+
+
+def validate_step_count_per_record(step_count_per_record):
+    if not is_int(step_count_per_record):
+        raise TypeError('step_count_per_record must be int.')
+    if step_count_per_record < 1:
+        raise ValueError("step_count_per_record must greater than 0")
+    if step_count_per_record > 1e6:
+        raise ValueError("step_count_per_record must smaller than 1e6")
+
+
+def validate_dynamic_on(dynamic_on):
+    if not isinstance(dynamic_on, bool):
+        raise TypeError('dynamic_on should be a bool')
+
+
+def validate_monitor_mbs_grad(monitor_mbs_grad):
+    if not isinstance(monitor_mbs_grad, bool):
+        logger.warning(f'monitor_mbs_grad should be a bool, actual value is {monitor_mbs_grad}.')
+        return False
+    return monitor_mbs_grad
+
+
+def validate_append_output(append_output):
+    if not isinstance(append_output, list):
+        raise TypeError('append_output should be a list')
+    if len(append_output) > 0 and len(append_output) != 2:
+        raise ValueError('append_output should be empty or contain exactly 2 elements')
+
+
+def validate_config(config):
+    config['ops'] = validate_ops(config.get('ops', []))
+
+    ndigits = config.get('ndigits')
+    validate_ndigits(ndigits)
+
+    eps = config.get('eps', 1e-8)
+    if not isinstance(eps, float):
+        raise TypeError("eps should be a float")
+
+    ranks = config.get("module_ranks", [])
+    validate_ranks(ranks)
+
+    targets = config.get("targets", {})
+    validate_targets(targets)
+
+    print_struct = config.get('print_struct', False)
+    validate_print_struct(print_struct)
+
+    ur_distribution = config.get('ur_distribution', False)
+    validate_ur_distribution(ur_distribution)
+
+    xy_distribution = config.get('xy_distribution', False)
+    validate_xy_distribution(xy_distribution)
+
+    wg_distribution = config.get('wg_distribution', False)
+    validate_wg_distribution(wg_distribution)
+
+    mg_distribution = config.get('mg_distribution', False)
+    validate_mg_distribution(mg_distribution)
+
+    param_distribution = config.get('param_distribution', False)
+    validate_param_distribution(param_distribution)
+
+    cc_distribution = config.get('cc_distribution', {})
+    validate_cc_distribution(cc_distribution)
+
+    alert = config.get('alert', {})
+    validate_alert(alert)
+
+    step_count_per_record = config.get('step_count_per_record', 1)
+    validate_step_count_per_record(step_count_per_record)
+
+    config["start_step"] = validate_int_arg(config.get("start_step"), "start_step",
+                                            MonitorConst.DEFAULT_START_STEP, MonitorConst.DEFAULT_START_STEP)
+    config["collect_times"] = validate_int_arg(config.get("collect_times"), "collect_times",
+                                               MonitorConst.DEFAULT_MIN_COLLECT_TIMES,
+                                               MonitorConst.DEFAULT_MAX_COLLECT_TIMES)
+    config["step_interval"] = validate_int_arg(config.get("step_interval"), "step_interval",
+                                               MonitorConst.DEFAULT_STEP_INTERVAL, MonitorConst.DEFAULT_STEP_INTERVAL)
+
+    squash_name = config.get('squash_name', True)
+    validate_squash_name(squash_name)
+
+    time_tags = config.get("append_output", [])
+    validate_append_output(time_tags)
+
+    config["monitor_mbs_grad"] = validate_monitor_mbs_grad(config.get('monitor_mbs_grad', False))
+
+    dynamic_on = config.get('dynamic_on', False)
+    validate_dynamic_on(dynamic_on)
+
+    if not targets:
+        if xy_distribution:
+            config["all_xy"] = True
+        config["targets"] = {"": {}}
+
+
+def time_str2time_digit(time_str):
+    time_format = '%b%d_%H-%M-%S'
+    if not isinstance(time_str, str):
+        raise TypeError(f"time_str:{time_str} should be a str")
+    try:
+        time_digit = datetime.strptime(time_str, time_format)
+    except Exception as e:
+        raise RuntimeError(f"illegal timestamp: {time_str}, timestamp should be prefix \
+                           of existing output dirpath, like 'Dec03_21-34-40'.") from e
+    return time_digit
+
+
+def get_target_output_dir(monitor_path, time_start, time_end):
+    check_file_or_directory_path(monitor_path, isdir=True)
+    time_start = time_str2time_digit(time_start) if time_start is not None else time_start
+    time_end = time_str2time_digit(time_end) if time_end is not None else time_end
+    if time_start and time_end and time_start > time_end:
+        raise ValueError(f"time_start({time_start}) greater than time_end({time_end})")
+    result = {}
+    for dirname in os.listdir(monitor_path):
+        match = re.match(MonitorConst.OUTPUT_DIR_PATTERN, dirname)
+        if not match:
+            continue
+        time_tag = match.group(1)
+        rank = match.group(2)
+        target_time = time_str2time_digit(time_tag)
+        start_ok = time_start is None or target_time >= time_start
+        end_ok = time_end is None or target_time <= time_end
+        if start_ok and end_ok:
+            result[rank] = os.path.join(monitor_path, dirname)
+    return result
+
+
+def chmod_tensorboard_dir(path):
+    """
+        format配置为tensorboard时，需要补充文件权限设置
+    """
+    try:
+        recursive_chmod(path)
+    except Exception as e:
+        logger.warning(f"chmod tensorboard dir wrong because {e}, not updated, please check!!!")
+
+
+def validate_set_monitor(grad_acc_steps, start_iteration):
+    """
+    validate parameters of set_monitor.
+    """
+    grad_acc_steps = validate_int_arg(grad_acc_steps, "grad_acc_steps",
+                                      MonitorConst.DEFAULT_GRAD_ACC_STEPS, MonitorConst.DEFAULT_GRAD_ACC_STEPS)
+
+    start_iteration = validate_int_arg(start_iteration, "start_iteration",
+                                       MonitorConst.DEFAULT_START_ITERATION, MonitorConst.DEFAULT_START_ITERATION)
+    return grad_acc_steps, start_iteration
+
+
+def validate_int_arg(value, name, minimum, default_value):
+    """Validate int args, if any exception occurs, use the default value."""
+    if value is None:
+        return default_value
+    try:
+        if not is_int(value):
+            raise TypeError(f"{name} must be int")
+        if value < minimum:
+            raise ValueError(f"{name} must greater than {minimum}")
+    except Exception as e:
+        value = default_value
+        logger.warning(f"Validate {name} failed, {e}, replaced with default value {value}.")
+    return value

msprobe/core/service.py

@@ -303,7 +303,8 @@ class BaseService(ABC):
                 self.logger.info(
                     f"Current token id: {self.cur_token_id}, exceed token_range, early stop dump infer token.")
             self.cur_token_id += 1
-        if isinstance(root_model, list):
+        # 此处root_model可以保证为 Module/Cell类型 或 [Module/Cell]类型
+        if root_model and isinstance(root_model, list):
             root_model = root_model[0]
             self.logger.warning("Infer model can only input one to support token_range, choose the first one.")
         if self._is_online_run_ut: