mindstudio-probe 1.0.3__py3-none-any.whl → 1.1.0__py3-none-any.whl

msprobe/core/common/inplace_op_checker.py

@@ -0,0 +1,38 @@
+import os
+from msprobe.core.common.file_utils import load_yaml
+
+
+class InplaceOpChecker:
+    OP_FUNCTIONAL = 'functional'
+    OP_TENSOR = 'tensor'
+    OP_TORCH = 'torch'
+    OP_DISTRIBUTED = 'distributed'
+
+    INPLACE_OPS_DICT = None
+
+    @classmethod
+    def load_ops(cls):
+        if cls.INPLACE_OPS_DICT is None:
+            cls.INPLACE_OPS_DICT = dict()
+        cur_path = os.path.dirname(os.path.realpath(__file__))
+        yaml_path = os.path.join(cur_path, "inplace_ops.yaml")
+        all_ops = load_yaml(yaml_path)
+        cls.INPLACE_OPS_DICT[cls.OP_FUNCTIONAL] = all_ops.get('inplace_functional_op')
+        cls.INPLACE_OPS_DICT[cls.OP_TENSOR] = all_ops.get('inplace_tensor_op')
+        cls.INPLACE_OPS_DICT[cls.OP_TORCH] = all_ops.get('inplace_torch_op')
+        cls.INPLACE_OPS_DICT[cls.OP_DISTRIBUTED] = all_ops.get('inplace_distributed_op')
+
+    @classmethod
+    def check(cls, api, category='distributed'):
+        """
+            给定api和分类，检查其是否为inplace操作
+        """
+        if not cls.INPLACE_OPS_DICT:
+            cls.load_ops()
+
+        if category not in cls.INPLACE_OPS_DICT.keys():
+            return False
+        return api in cls.INPLACE_OPS_DICT[category]
+
+
+InplaceOpChecker.load_ops()

msprobe/core/common/inplace_ops.yaml

@@ -0,0 +1,251 @@
+inplace_functional_op:
+  - threshold_
+  - relu_
+  - hardtanh_
+  - elu_
+  - selu_
+  - celu_
+  - leaky_relu_
+  - rrelu_
+
+inplace_tensor_op:
+  - __iadd__
+  - __iand__
+  - __idiv__
+  - __ifloordiv__
+  - __ilshift__
+  - __imod__
+  - __imul__
+  - __ior__
+  - __irshift__
+  - __isub__
+  - __ixor__
+  - abs_
+  - absolute_
+  - acos_
+  - acosh_
+  - add_
+  - addbmm_
+  - addcdiv_
+  - addcmul_
+  - addmm_
+  - addmv_
+  - addr_
+  - arccos_
+  - arccosh_
+  - arcsin_
+  - arcsinh_
+  - arctan_
+  - arctanh_
+  - asin_
+  - asinh_
+  - atan2_
+  - atan_
+  - atanh_
+  - baddbmm_
+  - bernoulli_
+  - bitwise_and_
+  - bitwise_not_
+  - bitwise_or_
+  - bitwise_xor_
+  - cauchy_
+  - ceil_
+  - clamp_
+  - clamp_max_
+  - clamp_min_
+  - clip_
+  - copysign_
+  - cos_
+  - cosh_
+  - cumprod_
+  - cumsum_
+  - deg2rad_
+  - digamma_
+  - div_
+  - divide_
+  - eq_
+  - erf_
+  - erfc_
+  - erfinv_
+  - exp2_
+  - exp_
+  - expm1_
+  - exponential_
+  - fill_
+  - fill_diagonal_
+  - fix_
+  - float_power_
+  - floor_
+  - floor_divide_
+  - fmod_
+  - frac_
+  - gcd_
+  - ge_
+  - geometric_
+  - greater_
+  - gt_
+  - greater_equal_
+  - heaviside_
+  - hypot_
+  - igamma_
+  - igammac_
+  - index_add_
+  - index_copy_
+  - index_fill_
+  - index_put_
+  - lcm_
+  - ldexp_
+  - le_
+  - lerp_
+  - less_
+  - less_equal_
+  - lgamma_
+  - log10_
+  - log1p_
+  - log2_
+  - log_
+  - log_normal_
+  - logical_and_
+  - logical_not_
+  - logical_or_
+  - logical_xor_
+  - logit_
+  - lt_
+  - map2_
+  - map_
+  - masked_fill_
+  - masked_scatter_
+  - mul_
+  - multiply_
+  - mvlgamma_
+  - ne_
+  - neg_
+  - negative_
+  - normal_
+  - not_equal_
+  - pow_
+  - polygamma_
+  - put_
+  - rad2deg_
+  - reciprocal_
+  - relu_
+  - remainder_
+  - renorm_
+  - resize_
+  - resize_as_
+  - round_
+  - rsqrt_
+  - scatter_
+  - scatter_add_
+  - sgn_
+  - sigmoid_
+  - sign_
+  - sin_
+  - sinc_
+  - sinh_
+  - sqrt_
+  - square_
+  - squeeze_
+  - sub_
+  - t_
+  - tan_
+  - tanh_
+  - transpose_
+  - tril_
+  - triu_
+  - true_divide_
+  - trunc_
+  - unsqueeze_
+  - xlogy_
+
+inplace_torch_op:
+  - _add_relu_
+  - abs_
+  - acos_
+  - acosh_
+  - addmv_
+  - alpha_dropout_
+  - arccos_
+  - arccosh_
+  - arcsin_
+  - arcsinh_
+  - arctan_
+  - arctanh_
+  - asin_
+  - asinh_
+  - atan_
+  - atanh_
+  - ceil_
+  - celu_
+  - clamp_
+  - clamp_max_
+  - clamp_min_
+  - clip_
+  - cos_
+  - cosh_
+  - deg2rad_
+  - dropout_
+  - embedding_renorm_
+  - erf_
+  - erfc_
+  - exp2_
+  - exp_
+  - expm1_
+  - feature_alpha_dropout_
+  - feature_dropout_
+  - fill_
+  - fix_
+  - floor_
+  - frac_
+  - gcd_
+  - index_put_
+  - lcm_
+  - ldexp_
+  - log10_
+  - log1p_
+  - log2_
+  - log_
+  - logit_
+  - nan_to_num_
+  - neg_
+  - negative_
+  - rad2deg_
+  - reciprocal_
+  - relu_
+  - resize_as_
+  - round_
+  - rrelu_
+  - rsqrt_
+  - selu_
+  - sigmoid_
+  - sin_
+  - sinc_
+  - sinh_
+  - sqrt_
+  - square_
+  - tan_
+  - tanh_
+  - threshold_
+  - trunc_
+  - xlogy_
+
+inplace_distributed_op:
+  - broadcast
+  - all_reduce
+  - reduce
+  - all_gather
+  - gather
+  - scatter
+  - reduce_scatter
+  - _reduce_scatter_base
+  - _all_gather_base
+  - send
+  - recv
+  - irecv
+  - isend
+  - all_to_all_single
+  - all_to_all
+  - all_gather_into_tensor
+  - reduce_scatter_tensor
+
+

msprobe/core/common/log.py

@@ -1,69 +1,86 @@
-import os
-import time
-import sys
-from functools import wraps
-from msprobe.core.common.const import MsgConst
-
-
-class BaseLogger:
-    def __init__(self):
-        self.warning_level = "WARNING"
-        self.error_level = "ERROR"
-        self.info_level = "INFO"
-        self.rank = None
-
-    @staticmethod
-    def _print_log(level, msg, end='\n'):
-        current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
-        pid = os.getpid()
-        full_msg = f"{current_time} ({pid}) [{level}] {msg}"
-        print(full_msg, end=end)
-        sys.stdout.flush()
-
-    def get_rank(self):
-        return self.rank
-
-    def filter_special_chars(func):
-        @wraps(func)
-        def func_level(self, msg):
-            for char in MsgConst.SPECIAL_CHAR:
-                msg = msg.replace(char, '_')
-            return func(self, msg)
-        return func_level
-
-    @filter_special_chars
-    def info(self, msg):
-        self._print_log(self.info_level, msg)
-
-    @filter_special_chars
-    def error(self, msg):
-        self._print_log(self.error_level, msg)
-
-    @filter_special_chars
-    def warning(self, msg):
-        self._print_log(self.warning_level, msg)
-
-    def on_rank_0(self, func):
-        def func_rank_0(*args, **kwargs):
-            current_rank = self.get_rank()
-            if current_rank is None or current_rank == 0:
-                return func(*args, **kwargs)
-            else:
-                return None
-        return func_rank_0
-
-    def info_on_rank_0(self, msg):
-        return self.on_rank_0(self.info)(msg)
-
-    def error_on_rank_0(self, msg):
-        return self.on_rank_0(self.error)(msg)
-
-    def warning_on_rank_0(self, msg):
-        return self.on_rank_0(self.warning)(msg)
-    
-    def error_log_with_exp(self, msg, exception):
-        self.error(msg)
-        raise exception
-
-
-logger = BaseLogger()
+import os
+import time
+import sys
+from functools import wraps
+from msprobe.core.common.const import MsgConst
+
+
+class BaseLogger:
+    def __init__(self):
+        self.rank = None
+        self.level = self.get_level()
+
+    @staticmethod
+    def get_level():
+        input_level = os.environ.get(MsgConst.MSPROBE_LOG_LEVEL)
+        if input_level not in MsgConst.LOG_LEVEL_ENUM:
+            return MsgConst.LogLevel.INFO.value
+        else:
+            return int(input_level)
+
+    def get_rank(self):
+        return self.rank
+
+    def filter_special_chars(func):
+        @wraps(func)
+        def func_level(self, msg, **kwargs):
+            for char in MsgConst.SPECIAL_CHAR:
+                msg = msg.replace(char, '_')
+            return func(self, msg, **kwargs)
+        return func_level
+    
+    @filter_special_chars
+    def error(self, msg):
+        if self.level <= MsgConst.LogLevel.ERROR.value:
+            self._print_log(MsgConst.LOG_LEVEL[3], msg)
+
+    @filter_special_chars
+    def warning(self, msg):
+        if self.level <= MsgConst.LogLevel.WARNING.value:
+            self._print_log(MsgConst.LOG_LEVEL[2], msg)
+
+    @filter_special_chars
+    def info(self, msg):
+        if self.level <= MsgConst.LogLevel.INFO.value:
+            self._print_log(MsgConst.LOG_LEVEL[1], msg)
+
+    @filter_special_chars
+    def debug(self, msg):
+        if self.level <= MsgConst.LogLevel.DEBUG.value:
+            self._print_log(MsgConst.LOG_LEVEL[0], msg)
+
+    def on_rank_0(self, func):
+        def func_rank_0(*args, **kwargs):
+            current_rank = self.get_rank()
+            if current_rank is None or current_rank == 0:
+                return func(*args, **kwargs)
+            else:
+                return None
+        return func_rank_0
+
+    def info_on_rank_0(self, msg):
+        return self.on_rank_0(self.info)(msg)
+
+    def error_on_rank_0(self, msg):
+        return self.on_rank_0(self.error)(msg)
+
+    def warning_on_rank_0(self, msg):
+        return self.on_rank_0(self.warning)(msg)
+    
+    def error_log_with_exp(self, msg, exception):
+        self.error(msg)
+        raise exception
+
+    def _print_log(self, level, msg, end='\n'):
+        current_rank = self.get_rank()
+        current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+        pid = os.getpid()
+        if current_rank is not None:
+            full_msg = f"{current_time} ({pid}) [rank {current_rank}] [{level}] {msg}"
+        else:
+            full_msg = f"{current_time} ({pid}) [{level}] {msg}"
+        print(full_msg, end=end)
+        sys.stdout.flush()
+
+
+logger = BaseLogger()