mindspore 2.1.0__cp37-none-any.whl → 2.2.11__cp37-none-any.whl

mindspore/ops/op_info_register.py

@@ -21,14 +21,225 @@ import inspect
 import json
 import os
 import functools
+import platform
+import hashlib
+import shutil
 
 from mindspore._c_expression import Oplib
 from mindspore import _checkparam as validator
+from mindspore import log as logger
+
+if platform.system() == "Linux":
+    import fcntl
 
 # path of built-in op info register.
 BUILT_IN_OPS_REGISTER_PATH = "mindspore/ops/_op_impl"
 BUILT_IN_CUSTOM_OPS_REGISTER_PATH = "mindspore/ops/_op_impl/_custom_op"
 
+KEY_NAME = "name"
+ASCEND_CUSTOM_OPP_PATH = "ASCEND_CUSTOM_OPP_PATH"
+
+
+def _get_reg_info_attr(op_info, attr_name, default_value=None):
+    """get attr value"""
+    for _, item in enumerate(op_info.get("attr", [])):
+        if item.get(KEY_NAME) == attr_name:
+            return item.get("defaultValue")
+    return default_value
+
+
+class _CustomInstaller:
+    """save custom op registration information to a json file which will be used by GE"""
+    reg_info_hash = []  # used to avoid writing the same reg info to file multiple times
+    copied_paths = []  # used to avoid copying the same file multiple times
+
+    def __init__(self, op_info, func=None):
+        self.op_info = op_info
+        self.func = func
+        self.op_type = op_info.get("op_name") if not func else func.__name__
+        vendor_name = "ms"
+        custom_dir = os.path.join(os.path.realpath("./"), "vendors", vendor_name)
+        self._set_env(custom_dir)
+        op_impl_dir = os.path.join(custom_dir, "op_impl")
+        self.ai_core_config_dir = os.path.join(op_impl_dir, "ai_core", "tbe", "config")
+        self.ai_core_impl_dir = os.path.join(op_impl_dir, "ai_core", "tbe", vendor_name + "_impl")
+        self.ai_cpu_config_dir = os.path.join(op_impl_dir, "cpu", "config")
+        self.ai_cpu_impl_dir = os.path.join(op_impl_dir, "cpu", "aicpu_kernel", "impl")
+
+    @staticmethod
+    def _set_env(custom_opp_path):
+        """set custom file path to env"""
+        if not os.environ.get(ASCEND_CUSTOM_OPP_PATH):
+            os.environ[ASCEND_CUSTOM_OPP_PATH] = custom_opp_path
+        else:
+            paths = os.environ[ASCEND_CUSTOM_OPP_PATH].split(':')
+            if custom_opp_path not in paths:
+                os.environ[ASCEND_CUSTOM_OPP_PATH] = custom_opp_path + ':' + os.environ[ASCEND_CUSTOM_OPP_PATH]
+
+    @staticmethod
+    def _create_dir(*dir_names):
+        """create directory"""
+        for dir_name in dir_names:
+            if not os.path.isdir(dir_name):
+                try:
+                    os.makedirs(dir_name, exist_ok=True)
+                except OSError as err:
+                    if err.errno == 17:  # File exists
+                        pass
+                    else:
+                        raise err
+
+    @staticmethod
+    def _copy_file(src_path, dst_dir):
+        """copy file"""
+        if not os.path.exists(src_path) or src_path in _CustomInstaller.copied_paths:
+            return
+        _CustomInstaller.copied_paths.append(src_path)
+        if os.path.isfile(src_path):
+            lock_file = os.path.join(dst_dir, "file.lock")
+            with os.fdopen(os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f:
+                fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+                shutil.copy(src_path, dst_dir)
+
+    def check(self):
+        """check if the reg info need written"""
+        if platform.system() != "Linux":
+            return False
+        if not os.environ.get("MS_DEV_CUSTOM_OPP_PATH"):
+            # only process the first time import the mindspore module
+            return False
+        if self.op_info.get("target") in ["GPU", "CPU"]:
+            return False
+        sha256 = hashlib.sha256()
+        value = json.dumps(self.op_info, sort_keys=True).encode()
+        sha256.update(value)
+        hash_value = sha256.hexdigest()
+        if hash_value in _CustomInstaller.reg_info_hash:
+            return False
+        _CustomInstaller.reg_info_hash.append(hash_value)
+        return True
+
+    def _find_ai_cpu_so_path(self, so_file):
+        """find the absolute path of so"""
+        current_path = os.path.dirname(os.path.abspath(__file__))
+        search_paths = [current_path + "/../lib", current_path + "/../lib/plugin/ascend"]
+        for path in search_paths:
+            so_path = os.path.join(path, so_file)
+            if os.path.exists(so_path):
+                return so_path
+        logger.warning("For Custom op '{}', can not find the aicpu so file '{}' in the following directories:\n{}"
+                       .format(self.op_type, so_file, "\n".join(search_paths)))
+        return ""
+
+    def _gen_ai_core_reg_info(self, imply_path, func_name):
+        """generate reg info"""
+
+        def _get_dtype_format(idx):
+            data_type = []
+            data_format = []
+            for _, dtype_format in enumerate(self.op_info.get("dtype_format", [])):
+                if not dtype_format[idx][0]:
+                    data_type = None
+                else:
+                    data_type.append(dtype_format[idx][0])
+                if not dtype_format[idx][1]:
+                    data_format = None
+                else:
+                    if dtype_format[idx][1] == "DefaultFormat":
+                        data_format.append("ND")
+                    else:
+                        data_format.append(dtype_format[idx][1])
+            return data_type, data_format
+
+        op_info = {"opFile": {"value": os.path.splitext(os.path.basename(imply_path))[0]},
+                   "opInterface": {"value": func_name}}
+        # attr
+        attrs_name = []
+        for _, item in enumerate(self.op_info.get("attr", [])):
+            attr_name = item.get(KEY_NAME)
+            attrs_name.append(attr_name)
+            key = "attr_" + attr_name
+            op_info[key] = {}
+            for k, v in item.items():
+                if k != KEY_NAME:
+                    op_info[key][k] = v
+        if attrs_name:
+            op_info["attr"] = {"list": ",".join(attrs_name)}
+        # input and output
+        inputs = self.op_info.get("inputs", [])
+        outputs = self.op_info.get("outputs", [])
+        input_num = len(inputs)
+        output_num = len(outputs)
+        for i in range(input_num + output_num):
+            item = inputs[i] if i < input_num else outputs[i - input_num]
+            key = "input" if i < input_num else "output"
+            key += str(item.get("index"))
+            op_info[key] = {KEY_NAME: item.get(KEY_NAME),
+                            "paramType": item.get("paramType", "required"),
+                            "shape": item.get("shape", "all")}
+            dtype, formats = _get_dtype_format(i)
+            if dtype:
+                op_info[key]["dtype"] = ",".join(dtype)
+            if formats:
+                op_info[key]["format"] = ",".join(formats)
+        return op_info
+
+    @staticmethod
+    def _gen_ai_cpu_reg_info(so_file):
+        """generate reg info"""
+        op_info = {"opInfo": {"computeCost": "100",
+                              "engine": "DNN_VM_AICPU",
+                              "flagAsync": "False",
+                              "flagPartial": "False",
+                              "functionName": "RunCpuKernel",
+                              "kernelSo": so_file,
+                              "opKernelLib": "CUSTAICPUKernel",
+                              "userDefined": "True"}}
+        return op_info
+
+    def _save_op_info(self, dst_dir, file_name, op_info):
+        """save op info file"""
+        repo = {}
+        save_path = os.path.join(dst_dir, file_name)
+        lock_file = os.path.join(dst_dir, "file.lock")
+        with os.fdopen(os.open(lock_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as f:
+            fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+            if os.path.isfile(save_path):
+                with open(save_path, 'r') as fr:
+                    json_str = fr.read()
+                json_str = "{}" if json_str == "" else json_str
+                repo = json.loads(json_str)
+            repo.update({self.op_type: op_info})
+            with os.fdopen(os.open(save_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), 'w') as fw:
+                json.dump(repo, fw, sort_keys=True, indent=4, separators=(',', ':'))
+
+    def run(self):
+        """save reg info to file"""
+        if not self.check():
+            return
+        so_name = _get_reg_info_attr(self.op_info, "cust_aicpu")
+        if so_name:
+            _CustomInstaller._create_dir(self.ai_cpu_config_dir, self.ai_cpu_impl_dir)
+            # copy so file
+            so_file = "lib" + so_name + ".so"
+            imply_path = self._find_ai_cpu_so_path(so_file)
+            self._copy_file(imply_path, self.ai_cpu_impl_dir)
+            # generate and copy reg info file
+            op_info = self._gen_ai_cpu_reg_info(so_file)
+            self._save_op_info(self.ai_cpu_config_dir, "cust_aicpu_kernel.json", op_info)
+        else:
+            _CustomInstaller._create_dir(self.ai_core_config_dir, self.ai_core_impl_dir)
+            # copy dsl file
+            imply_path = os.path.realpath(inspect.getfile(self.func))
+            self._copy_file(imply_path, self.ai_core_impl_dir)
+            # generate and copy reg info file
+            op_info = self._gen_ai_core_reg_info(imply_path, self.func.__name__)
+            self._copy_file(imply_path, self.ai_core_impl_dir)
+            for arc_name in ["ascend910", "ascend910b"]:
+                arc_dir = os.path.join(self.ai_core_config_dir, arc_name)
+                _CustomInstaller._create_dir(arc_dir)
+                self._save_op_info(arc_dir, "aic-{}-ops-info.json".format(arc_name), op_info)
+
 
 def op_info_register(op_info):
     r"""
@@ -125,6 +336,12 @@ def custom_info_register(*reg_info):
 
     def decorator(func):
         setattr(func, "reg_info", reg_info)
+        if reg_info:
+            used_reg_info = reg_info[0]
+            if isinstance(used_reg_info, dict):
+                # ai_cpu should be parsed inside CustomRegOp, skip it here
+                if not _get_reg_info_attr(used_reg_info, "cust_aicpu"):
+                    _CustomInstaller(used_reg_info, func).run()
 
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
@@ -140,7 +357,7 @@ class RegOp:
     Base class for op info register.
 
     Args:
-        op_name (str): Name of op.
+        op_name (str): Name of operator.
     """
 
     def __init__(self, op_name=""):
@@ -167,7 +384,6 @@ class RegOp:
         """
         if not isinstance(value, str):
             raise TypeError("%s value must be str" % str(value))
-        return True
 
     def _is_int(self, value):
         """
@@ -181,7 +397,6 @@ class RegOp:
         """
         if not isinstance(value, int):
             raise TypeError("%s value must be int" % str(value))
-        return True
 
     def _is_bool(self, value):
         """
@@ -195,7 +410,6 @@ class RegOp:
         """
         if not isinstance(value, bool):
             raise TypeError("%s value must be bool" % str(value))
-        return True
 
     @staticmethod
     def _is_list(value):
@@ -210,7 +424,6 @@ class RegOp:
         """
         if not isinstance(value, list):
             raise TypeError("%s value must be list" % str(value))
-        return True
 
     def _check_param(self, param_list, key_list, fn_list, kwargs):
         """
@@ -278,7 +491,9 @@ class RegOp:
             self._is_string(arg[1])
             if len(arg) == 3:
                 self._is_string(arg[2])
-            dtype_format.append(arg)
+                dtype_format.append(arg)
+            else:
+                dtype_format.append(arg)
         self.dtype_format_.append(tuple(dtype_format))
         return self
 
@@ -446,10 +661,10 @@ class AkgCpuRegOp(AkgRegOp):
 
 class AiCPURegOp(CpuRegOp):
     r"""
-    Class for AiCPU operator information register.
+    Class for AiCPU operator information registration.
 
     Args:
-        op_name (str):kernel name.
+        op_name (str): Name of operator.
 
     Examples:
         >>> from mindspore.ops import AiCPURegOp, DataType
@@ -481,14 +696,15 @@ class AiCPURegOp(CpuRegOp):
 
 class TBERegOp(RegOp):
     r"""
-    Class for TBE operator information register.
+    Class for TBE operator information registration. TBE (Tensor Boost Engine) is the Ascend operator development
+    tool, which is extended on the basis of the TVM framework to develop custom operators.
 
     Args:
-        op_name (str):kernel name.
+        op_name (str): Name of operator.
 
     Examples:
-        >>> import mindspore.ops as ops
-        >>> op_name_op_info = ops.TBERegOp("OpName") \
+        >>> from mindspore.ops import TBERegOp, DataType
+        >>> op_name_op_info = TBERegOp("OpName") \
         ...    .fusion_type("ELEMWISE") \
         ...    .async_flag(False) \
         ...    .binfile_name("op_name.so") \
@@ -505,14 +721,14 @@ class TBERegOp(RegOp):
         ...    .input(0, "x2", None, "required", None) \
         ...    .input(1, "axis", None, "required", None) \
         ...    .output(0, "y", True, "required", "all") \
-        ...    .real_input_index([1, 0])
-        ...    .input_to_attr_index([2])
-        ...    .unknown_shape_formats(["ND", "ND", "ND", "ND"])
+        ...    .real_input_index([1, 0]) \
+        ...    .input_to_attr_index([2]) \
+        ...    .unknown_shape_formats(["ND", "ND", "ND", "ND"]) \
         ...    .reshape_type("NC") \
         ...    .is_dynamic_format(True) \
-        ...    .dtype_format(DataType.F16_None, DataType.F16_None) \
-        ...    .dtype_format(DataType.F32_None, DataType.F32_None) \
-        ...    .dtype_format(DataType.I32_None, DataType.I32_None) \
+        ...    .dtype_format(DataType.F16_None, DataType.F16_None, DataType.F16_None, DataType.F16_None) \
+        ...    .dtype_format(DataType.F32_None, DataType.F32_None, DataType.F32_None, DataType.F32_None) \
+        ...    .dtype_format(DataType.I32_None, DataType.I32_None, DataType.I32_None, DataType.I32_None) \
         ...    .get_op_info()
         >>>
     """
@@ -706,7 +922,8 @@ class TBERegOp(RegOp):
         Args:
             pattern (str): Value of op pattern, e.g. "broadcast", "reduce". Default: ``None`` .
         """
-        if pattern is not None and self._is_string(pattern):
+        if pattern is not None:
+            self._is_string(pattern)
             self.op_pattern_ = pattern
         return self
 
@@ -830,7 +1047,7 @@ class CustomRegOp(RegOp):
 
         Tutorial Examples:
             - `Custom Operators (Custom-based) - Defining Custom Operator of aicpu Type
-              <https://mindspore.cn/tutorials/experts/en/r2.1/operation/op_custom.html#
+              <https://mindspore.cn/tutorials/experts/en/r2.2/operation/op_custom.html#
               defining-custom-operator-of-aicpu-type>`_
         """
         param_list = [index, name, param_type]
@@ -870,7 +1087,7 @@ class CustomRegOp(RegOp):
 
         Tutorial Examples:
             - `Custom Operators (Custom-based) - Defining Custom Operator of aicpu Type
-              <https://mindspore.cn/tutorials/experts/en/r2.1/operation/op_custom.html#
+              <https://mindspore.cn/tutorials/experts/en/r2.2/operation/op_custom.html#
               defining-custom-operator-of-aicpu-type>`_
         """
         param_list = [index, name, param_type]
@@ -898,7 +1115,7 @@ class CustomRegOp(RegOp):
 
         Tutorial Examples:
             - `Custom Operators (Custom-based) - Defining Custom Operator of aicpu Type
-              <https://mindspore.cn/tutorials/experts/en/r2.1/operation/op_custom.html#
+              <https://mindspore.cn/tutorials/experts/en/r2.2/operation/op_custom.html#
               defining-custom-operator-of-aicpu-type>`_
         """
         io_nums = len(self.inputs) + len(self.outputs)
@@ -955,7 +1172,7 @@ class CustomRegOp(RegOp):
 
         Tutorial Examples:
             - `Custom Operators (Custom-based) - Defining Custom Operator of aicpu Type
-              <https://mindspore.cn/tutorials/experts/en/r2.1/operation/op_custom.html#
+              <https://mindspore.cn/tutorials/experts/en/r2.2/operation/op_custom.html#
               defining-custom-operator-of-aicpu-type>`_
         """
         param_list = [name, param_type, value_type, default_value]
@@ -981,7 +1198,7 @@ class CustomRegOp(RegOp):
 
         Tutorial Examples:
             - `Custom Operators (Custom-based) - Defining Custom Operator of aicpu Type
-              <https://mindspore.cn/tutorials/experts/en/r2.1/operation/op_custom.html#
+              <https://mindspore.cn/tutorials/experts/en/r2.2/operation/op_custom.html#
               defining-custom-operator-of-aicpu-type>`_
         """
         if target is not None:
@@ -996,7 +1213,7 @@ class CustomRegOp(RegOp):
 
         Tutorial Examples:
             - `Custom Operators (Custom-based) - Defining Custom Operator of aicpu Type
-              <https://mindspore.cn/tutorials/experts/en/r2.1/operation/op_custom.html#
+              <https://mindspore.cn/tutorials/experts/en/r2.2/operation/op_custom.html#
               defining-custom-operator-of-aicpu-type>`_
         """
         op_info = {}
@@ -1004,6 +1221,8 @@ class CustomRegOp(RegOp):
             if isinstance(k, str) and k.endswith('_'):
                 k = k.rstrip('_')
             op_info[k] = v
+        if _get_reg_info_attr(op_info, "cust_aicpu"):
+            _CustomInstaller(op_info).run()
         return op_info
 
 

mindspore/ops/operations/__init__.py

@@ -21,7 +21,7 @@ A collection of operators to build neural networks or to compute functions.
 
 from ._embedding_cache_ops import (CacheSwapTable, UpdateCache, MapCacheIdx, SubAndFilter,
                                    MapUniform, DynamicAssign, PadAndShift)
-from ._inner_ops import (MatmulDDS, DSDMatmul, Cummin, ExtractImagePatches)
+from ._inner_ops import (MatmulDDS, DSDMatmul, Cummin, ExtractImagePatches, SelectView, CopyWithSlice)
 from ._quant_ops import *
 from ._thor_ops import (CusBatchMatMul, CusCholeskyTrsm, CusFusedAbsMax1, CusImg2Col, CusMatMulCubeDenseLeft,
                         CusMatMulCubeFraczRightMul, CusMatMulCube, CusMatrixCombine, CusTranspose02314,
@@ -29,20 +29,20 @@ from ._thor_ops import (CusBatchMatMul, CusCholeskyTrsm, CusFusedAbsMax1, CusImg
                         LoadIm2Col, UpdateThorGradient, CholeskyTrsm,
                         DetTriangle, ProdForceSeA)
 from ._ms_kernel import (ms_kernel, kernel)
-from .array_ops import (ArgMaxWithValue, ArgMinWithValue, Argmax, Argmin, BatchToSpace, BatchToSpaceND,
+from .array_ops import (ArgMaxWithValue, ArgMinWithValue, Argmax, Argmin, BatchToSpace,
                         BatchToSpaceNDV2, BroadcastTo, Cast, Coalesce, Concat, Cummax, DType, DepthToSpace, Diag,
-                        DiagPart, DynamicShape, EditDistance, EmbeddingLookup, ExpandDims, ExtractVolumePatches,
-                        Eye, Fill, Gather, GatherD, GatherNd, GatherV2, Identity, Im2Col, InvertPermutation,
+                        DiagPart, EditDistance, EmbeddingLookup, ExpandDims, ExtractVolumePatches,
+                        Eye, Fill, Gather, GatherD, GatherNd, Identity, Im2Col, InvertPermutation,
                         LowerBound, Lstsq, MaskedFill, MaskedSelect, Meshgrid, Mvlgamma, Ones, OnesLike,
-                        Pack, Padding, ParallelConcat, PopulationCount, Range, Rank, Reshape, ResizeNearestNeighbor,
-                        ReverseSequence, ReverseV2, Rint, ScalarToArray, ScalarToTensor, ScatterAdd,
+                        Padding, ParallelConcat, PopulationCount, Range, Rank, Reshape, ResizeNearestNeighbor,
+                        ReverseSequence, ReverseV2, Rint, ScalarToTensor, ScatterAdd,
                         ScatterDiv, ScatterMax, ScatterMin, ScatterMul, ScatterNd, ScatterNdAdd, ScatterNdDiv,
-                        ScatterNdMax, ScatterNdMin, ScatterNdSub, ScatterNdUpdate, ScatterNonAliasingAdd, ScatterSub,
+                        ScatterNdMax, ScatterNdMin, ScatterNdSub, ScatterNdUpdate, ScatterSub,
                         ScatterUpdate, SearchSorted, Select, Shape, Size, Slice, Sort, SpaceToBatch, SpaceToBatchND,
                         SpaceToDepth, SparseGatherV2, Split, SplitV, Squeeze, Stack, StridedSlice, TensorScatterAdd,
                         TensorScatterDiv, TensorScatterMax, TensorScatterMin, TensorScatterMul, TensorScatterSub,
                         TensorScatterUpdate, TensorShape, Tile, TopK, TransShape, Transpose, TupleToArray, Unique,
-                        UniqueWithPad, Unpack, UnsortedSegmentMax, UnsortedSegmentMin, UnsortedSegmentProd,
+                        UniqueWithPad, UnsortedSegmentMax, UnsortedSegmentMin, UnsortedSegmentProd,
                         UnsortedSegmentSum, Unstack, UpperBound, Zeros, ZerosLike, AffineGrid, Bincount, CheckNumerics,
                         HammingWindow, IdentityN, IndexFill, LeftShift, ListDiff, LogSpace, MatrixBandPart,
                         MatrixDiagPartV3, MatrixDiagV3, MatrixSetDiagV3, NonZero, Expand, Col2Im, ConjugateTranspose,
@@ -69,7 +69,7 @@ from .inner_ops import (ScalarCast, Randperm, NoRepeatNGram, LambApplyOptimizerA
 from .linalg_ops import (Svd, Geqrf)
 from .math_ops import (Abs, ACos, Asin, Asinh, AddN, AccumulateNV2, AssignAdd, AssignSub, Atan2, BatchMatMul,
                        BitwiseAnd, BitwiseOr, Ger, BitwiseXor, Inv, Invert, ApproximateEqual,
-                       InplaceAdd, InplaceSub, InplaceUpdate, InplaceUpdateV2,
+                       InplaceAdd, InplaceSub, InplaceUpdateV2,
                        ReduceMax, ReduceMin, ReduceMean, ReduceSum, ReduceAll, ReduceProd, CumProd, Cdist, ReduceAny,
                        Cos, Cross, Div, DivNoNan, Equal, EqualCount, Exp, Expm1, Erf, Erfc, Floor, FloorDiv, FloorMod,
                        Ceil, Acosh, Greater, GreaterEqual, Lerp, Less, LessEqual, Log, Log1p, LogicalAnd, Mod,
@@ -79,7 +79,7 @@ from .math_ops import (Abs, ACos, Asin, Asinh, AddN, AccumulateNV2, AssignAdd, A
                        NPUGetFloatStatus, Pow, RealDiv, IsNan, IsInf, IsFinite, FloatStatus,
                        Reciprocal, CumSum, HistogramFixedWidth, SquaredDifference, Xdivy, Xlogy,
                        Sin, Sqrt, Rsqrt, BesselI0e, BesselI1e, TruncateDiv, TruncateMod, Addcdiv,
-                       Addcmul, Square, Sub, TensorAdd, Add, Sign, Round, SquareSumAll, Atan, Atanh, Cosh, Sinh, Eps,
+                       Addcmul, Square, Sub, Add, Sign, Round, SquareSumAll, Atan, Atanh, Cosh, Sinh, Eps,
                        Tan, MatrixInverse, IndexAdd, Erfinv, Conj, Real, Imag, Complex, Trunc, IsClose, LuSolve,
                        CholeskyInverse, BesselJ0, BesselJ1, BesselK0, BesselK0e, BesselK1, BesselK1e, BesselY0,
                        BesselY1, Bucketize, Cauchy, Cholesky, CholeskySolve, Betainc,
@@ -92,14 +92,14 @@ from .math_ops import (Abs, ACos, Asin, Asinh, AddN, AccumulateNV2, AssignAdd, A
 from .nn_ops import (LSTM, SGD, Adam, AdamWeightDecay, FusedSparseAdam, FusedSparseLazyAdam, AdamNoUpdateParam,
                      ApplyMomentum, BatchNorm, BiasAdd, Conv2D, Conv3D, Conv2DTranspose, Conv3DTranspose,
                      DepthwiseConv2dNative,
-                     DropoutDoMask, Dropout, Dropout2D, Dropout3D, DropoutGenMask, Flatten,
-                     InstanceNorm, BNTrainingReduce, BNTrainingUpdate,
-                     GeLU, Gelu, FastGeLU, FastGelu, Elu, CeLU,
+                     Dropout, Dropout2D, Dropout3D, Flatten,
+                     InstanceNorm,
+                     GeLU, FastGeLU, Elu, CeLU,
                      GetNext, L2Normalize, LayerNorm, L2Loss, CTCLoss, CTCLossV2, CTCLossV2Grad, CTCGreedyDecoder,
                      LogSoftmax, MaxPool3D, AvgPool3D,
                      MaxPool, DataFormatDimMap,
                      AvgPool, Conv2DBackpropInput, ComputeAccidentalHits,
-                     MaxPoolWithArgmax, MaxPoolWithArgmaxV2, OneHot, Pad, MirrorPad, Mish, PReLU, ReLU, ReLU6, ReLUV2,
+                     MaxPoolWithArgmaxV2, OneHot, Pad, MirrorPad, Mish, PReLU, ReLU, ReLU6, ReLUV2,
                      HSwish, HSigmoid,
                      ResizeBilinear, Sigmoid, SeLU, HShrink, ApplyKerasMomentum,
                      SigmoidCrossEntropyWithLogits, NLLLoss, BCEWithLogitsLoss,
@@ -115,13 +115,13 @@ from .nn_ops import (LSTM, SGD, Adam, AdamWeightDecay, FusedSparseAdam, FusedSpa
                      ApplyAdamWithAmsgrad, ApplyAdamWithAmsgradV2, AdaptiveAvgPool3D, AdaptiveMaxPool2D,
                      AdaptiveMaxPool3D,
                      GridSampler3D, MaxPool3DWithArgmax, MaxUnpool2D, NuclearNorm, NthElement, MultilabelMarginLoss,
-                     Dilation2D, DataFormatVecPermute, DeformableOffsets, FractionalAvgPool,
+                     Dilation2D, DataFormatVecPermute, DeformableOffsets, Dense, FractionalAvgPool,
                      FractionalMaxPool, FractionalMaxPool3DWithFixedKsize, FractionalMaxPoolWithFixedKsize,
                      GridSampler2D, TripletMarginLoss, UpsampleNearest3D, UpsampleTrilinear3D, PadV3, ChannelShuffle,
-                     GLU, MaxUnpool3D, Pdist)
+                     GLU, MaxUnpool3D, Pdist, RmsNorm, PagedAttention, PagedAttentionMask, ReshapeAndCache)
 from .other_ops import (Assign, IOU, BoundingBoxDecode, BoundingBoxEncode,
                         ConfusionMatrix, UpdateState, Load, StopGradient,
-                        CheckValid, Partial, Depend, identity, Push, Pull, PyExecute, PyFunc, _DynamicLossScale,
+                        CheckValid, Partial, Depend, Push, Pull, PyExecute, PyFunc, _DynamicLossScale,
                         SampleDistortedBoundingBoxV2)
 from .random_ops import (RandomChoiceWithMask, StandardNormal, Gamma, RandomGamma, Poisson, UniformInt, UniformReal,
                          RandomCategorical, StandardLaplace, Multinomial, UniformCandidateSampler,
@@ -129,8 +129,13 @@ from .random_ops import (RandomChoiceWithMask, StandardNormal, Gamma, RandomGamm
                          ParameterizedTruncatedNormal, RandomPoisson, MultinomialWithReplacement, RandomShuffle,
                          RandpermV2)
 from .rl_ops import (BufferAppend, BufferGetItem, BufferSample)
-from .sparse_ops import (SparseToDense, SparseTensorDenseMatmul, SparseTensorDenseAdd, SparseSlice)
+from .sparse_ops import (
+    SparseToDense, SparseTensorDenseMatmul, SparseTensorDenseAdd, SparseSlice)
 from .spectral_ops import (BartlettWindow, BlackmanWindow)
+from ..deprecated import (identity, DropoutDoMask, MaxPoolWithArgmax,
+                          BNTrainingReduce, BNTrainingUpdate, DropoutGenMask, Gelu, FastGelu,
+                          TensorAdd, InplaceUpdate, ScatterNonAliasingAdd,
+                          BatchToSpaceND, Unpack, GatherV2, DynamicShape, ScalarToArray, Pack)
 
 __all__ = [
     'HSVToRGB',
@@ -616,7 +621,10 @@ __all__ = [
     "CumulativeLogsumexp",
     "DataFormatVecPermute",
     "DeformableOffsets",
+    "Dense",
     "ExtractImagePatches",
+    "SelectView",
+    "CopyWithSlice",
     "FillDiagonal",
     "Fills",
     "Gcd",
@@ -683,7 +691,11 @@ __all__ = [
     "IndexPut",
     "MaskedScatter",
     "Ormqr",
-    "RandpermV2"
+    "RandpermV2",
+    "RmsNorm",
+    "PagedAttention",
+    "PagedAttentionMask",
+    "ReshapeAndCache"
 ]
 
 __custom__ = [