mindspore 2.7.0rc1__cp310-cp310-win_amd64.whl → 2.7.1__cp310-cp310-win_amd64.whl

mindspore/_extends/remote/kernel_build_server_ascend.py

@@ -1,75 +0,0 @@
-# Copyright 2020-2021 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""kernel build server for ascend"""
-import sys
-import warnings
-import json
-
-from mindspore._extends.parallel_compile.tbe_compiler.tbe_job_manager import TbeJobManager
-from mindspore._extends.remote.kernel_build_server import Messager, get_logger, AkgBuilder
-
-
-class AscendMessager(Messager):
-    """
-    Ascend Messager
-    It works as a server, communicating with c++ client.
-    """
-
-    def __init__(self, fdin, fdout):
-        super().__init__(fdin, fdout)
-        get_logger().info("[TRACE] Ascend Messager init...")
-        self.tbe_builder = TbeJobManager()
-        self.akg_builder = AkgBuilder("ASCEND")
-
-    def handle(self):
-        """
-        Communicate with remote client.
-        Reference protocol between them at PR#3821 and PR#3935
-        """
-        arg = self.get_message()
-        if arg.startswith('AKG'):
-            self.akg_builder.handle(self, arg)
-        else:
-            job_json = dict()
-            try:
-                job_json = json.loads(arg)
-            except json.decoder.JSONDecodeError:
-                get_logger().error("[TRACE] Request is not a json message: {}".format(arg))
-                self.send_ack(False)
-                self.exit()
-            finally:
-                pass
-
-            if "job_type" in job_json:
-                res = self.tbe_builder.job_handler(arg)
-                self.send_res(res)
-            else:
-                get_logger().error("[TRACE] Request is not a TBE Job message: {}".format(arg))
-                self.send_ack(False)
-                self.exit()
-
-    def exit(self):
-        self.tbe_builder.reset()
-        get_logger().info("[TRACE] Ascend Messager Exit...")
-        exit()
-
-
-if __name__ == '__main__':
-    warnings.simplefilter("ignore")
-    if len(sys.argv) != 3:
-        raise Exception('Incorrect argv: {}'.format(sys.argv))
-    get_logger().debug(f"[TRACE] argv: {str(sys.argv)}")
-    messager = AscendMessager(int(sys.argv[1]), int(sys.argv[2]))
-    messager.run()

mindspore/communication/_hccl_management.py

@@ -1,297 +0,0 @@
-# Copyright 2020 Huawei Technologies Co., Ltd
-
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""HCCL management API"""
-from __future__ import absolute_import
-from __future__ import division
-
-import ctypes
-import os
-
-from mindspore import context
-from mindspore._c_expression import get_hccl_rank_id, get_hccl_rank_size
-
-MAX_GROUP_NAME_LEN = 127
-MAX_RANK_NUM = 4096
-HCCL_LIB = 'libhccl_plugin.so'
-HCCL_LIB_CTYPES = ""
-
-
-def check_group(group):
-    """
-    A function that check if a collection communication group is legal.
-
-    Returns:
-        None
-    """
-    if isinstance(group, (str)):
-        group_len = len(group)
-        if group_len > MAX_GROUP_NAME_LEN or group_len == 0:
-            raise ValueError("The length of communication group name must be in range [1, 127), "
-                             "but got the value : {} ".format(group_len))
-    else:
-        raise TypeError("The type of communication group name must be type of string, "
-                        "but got 'group' type : {}.".format(type(group)))
-
-
-def check_rank_num(rank_num):
-    """
-    A function that check if a collection communication rank number is legal.If not raise error.
-
-    Returns:
-        None
-    """
-    if isinstance(rank_num, (int)):
-        if rank_num > MAX_RANK_NUM or rank_num <= 0:
-            raise ValueError("For 'create_group', the size of argument 'rand_ids' should be greater than 0 and"
-                             "less than {}, but got the size of 'rank_ids' : {}.".format(MAX_RANK_NUM, rank_num))
-    else:
-        raise TypeError("The argument 'rank_num' must be type of int, "
-                        "but got 'rank_num' type : {}.".format(type(rank_num)))
-
-
-def check_rank_id(rank_id):
-    """
-    A function that check if a collection communication rank id is legal.If not raise error.
-
-    Returns:
-        None
-    """
-    if isinstance(rank_id, (int)):
-        if rank_id >= MAX_RANK_NUM or rank_id < 0:
-            raise ValueError("The rand id in the communication group must be greater or equal 0 and "
-                             "less than {}, but got type value : {}.".format(MAX_RANK_NUM, rank_id))
-    else:
-        raise TypeError("The rand id in the communication group must be must be type of int, "
-                        "but got type value : {}.".format(type(rank_id)))
-
-
-def load_lib():
-    """load hccl lib"""
-    try:
-        base_dir = os.path.dirname(os.path.realpath(__file__))
-        lib_path = os.path.join(base_dir, "../lib/plugin/ascend", HCCL_LIB)
-        hccl_lib = ctypes.CDLL(lib_path)
-    except Exception:
-        raise RuntimeError('Get hccl lib error.')
-
-    global HCCL_LIB_CTYPES
-    HCCL_LIB_CTYPES = hccl_lib
-
-
-def c_str(string):
-    """Convert a python string to C string."""
-    if not isinstance(string, str):
-        string = string.decode('ascii')
-    return ctypes.c_char_p(string.encode('utf-8'))
-
-
-def c_array(ctype, values):
-    """Create ctypes array from a python array."""
-    return (ctype * len(values))(*values)
-
-
-def create_group(group, rank_num, rank_ids):
-    """
-    Create group.
-
-    A function that creates a collection communication group which includes 'rank_num'
-    device and 'rank_ids' is the list of these ranks of devices.
-
-    Note:
-        The world group can not be created.
-
-    Returns:
-        None
-    """
-    check_group(group)
-    check_rank_num(rank_num)
-    if isinstance(rank_ids, (list)):
-        if rank_num != len(rank_ids):
-            raise ValueError("The argument 'rank_num' number should be equal to the length "
-                             "of rank_ids, but got 'rank_num' value : {} and 'rank_ids' value : {}."
-                             .format(rank_num, rank_ids))
-        for rank_id in rank_ids:
-            if not isinstance(rank_id, (int)) or rank_id < 0:
-                raise ValueError("The elements of argument 'rank_ids' must be "
-                                 "unsigned integer, but got the type : {}".format(type(rank_id)))
-        c_array_rank_ids = c_array(ctypes.c_uint, rank_ids)
-        c_rank_num = ctypes.c_uint(rank_num)
-        c_group = c_str(group)
-        ret = HCCL_LIB_CTYPES.HcomCreateGroup(c_group, c_rank_num, c_array_rank_ids)
-        if ret != 0:
-            raise RuntimeError('Create group error, the error code is {}.'.format(ret))
-    else:
-        raise TypeError("For 'create_group', the argument 'rank_ids' must be type of list, "
-                        "but got 'rank_ids' type : {}.".format(type(rank_ids)))
-
-
-def destroy_group(group):
-    """
-    A function that destroy the group which created by user.
-
-    Note:
-        The world group can not be destroy.
-
-    Returns:
-        None
-    """
-    check_group(group)
-    c_group = c_str(group)
-    ret = HCCL_LIB_CTYPES.HcomDestroyGroup(c_group)
-    if ret != 0:
-        raise RuntimeError('Destroy group error.')
-
-
-def get_rank_size(group="hccl_world_group"):
-    """
-    A function that returns the number of ranks within the given collection communication group.
-
-    Note:
-        The default group is hccl_world_group.
-
-    Returns:
-        An integer scalar with the num of ranks.
-    """
-
-    if context.get_context("mode") == context.PYNATIVE_MODE:
-        return get_hccl_rank_size()
-
-    check_group(group)
-    c_group = c_str(group)
-    c_rank_size = ctypes.c_uint()
-    ret = HCCL_LIB_CTYPES.HcomGetRankSize(c_group, ctypes.byref(c_rank_size))
-    if ret != 0:
-        raise RuntimeError('Get rank size error.')
-
-    return c_rank_size.value
-
-
-def get_rank_id(group="hccl_world_group"):
-    """
-    A function that returns the rank id of the calling process, within the given collection communication group.
-
-    Returns:
-        An integer scalar with the rank id of the calling process.
-    """
-
-    if context.get_context("mode") == context.PYNATIVE_MODE:
-        return get_hccl_rank_id()
-
-    check_group(group)
-    c_group = c_str(group)
-    c_rank_id = ctypes.c_uint()
-    ret = HCCL_LIB_CTYPES.HcomGetRankId(c_group, ctypes.byref(c_rank_id))
-    if ret != 0:
-        raise RuntimeError('Get rank id error.')
-
-    return c_rank_id.value
-
-
-
-def get_local_rank_size(group="hccl_world_group"):
-    """
-    A function that returns the number of local ranks within the given collection communication group.
-
-    Note:
-        The default group is hccl_world_group.
-
-    Returns:
-        An integer scalar with the num of local ranks.
-    """
-    if context.get_context("mode") is context.PYNATIVE_MODE:
-        raise RuntimeError("The function 'get_local_rank_size' is not supported in PYNATIVE_MODE, "
-                           "'get_local_rank_size' only support GRAPH_MODE")
-    check_group(group)
-    c_group = c_str(group)
-    c_local_rank_size = ctypes.c_uint()
-    ret = HCCL_LIB_CTYPES.HcomGetLocalRankSize(c_group, ctypes.byref(c_local_rank_size))
-    if ret != 0:
-        raise RuntimeError('Get local rank size error.')
-
-    return c_local_rank_size.value
-
-
-def get_local_rank_id(group="hccl_world_group"):
-    """
-    Get local rank id.
-
-    A function that returns the local rank id of the calling process, within the given collection communication group.
-
-    Returns:
-        An integer scalar with the local rank id of the calling process.
-    """
-
-    if context.get_context("mode") is context.PYNATIVE_MODE:
-        raise RuntimeError("The function 'get_local_rank_id' is not supported in PYNATIVE_MODE, "
-                           "'get_local_rank_id' only support GRAPH_MODE")
-    check_group(group)
-    c_group = c_str(group)
-    c_local_rank_id = ctypes.c_uint()
-    ret = HCCL_LIB_CTYPES.HcomGetLocalRankId(c_group, ctypes.byref(c_local_rank_id))
-    if ret != 0:
-        raise RuntimeError('Get local rank id error.')
-
-    return c_local_rank_id.value
-
-
-def get_world_rank_from_group_rank(group, group_rank_id):
-    """
-    Get world rank from group rank.
-
-    A function that returns the rank id in the world group corresponding to the
-    rank which id is 'group_rank_id' in the user group.
-
-    Returns:
-        An integer scalar with the rank id in the world group.
-    """
-    if context.get_context("mode") is context.PYNATIVE_MODE:
-        raise RuntimeError("The function 'get_world_rank_from_group_rank' is not supported in PYNATIVE_MODE, "
-                           "'get_world_rank_from_group_rank' only support GRAPH_MODE")
-    check_group(group)
-    check_rank_id(group_rank_id)
-    c_group = c_str(group)
-    c_group_rank_id = ctypes.c_uint(group_rank_id)
-    c_world_rank_id = ctypes.c_uint()
-    ret = HCCL_LIB_CTYPES.HcomGetWorldRankFromGroupRank(c_group, c_group_rank_id, ctypes.byref(c_world_rank_id))
-    if ret != 0:
-        raise RuntimeError('Get world rank from group rank error.')
-
-    return c_world_rank_id.value
-
-
-def get_group_rank_from_world_rank(world_rank_id, group):
-    """
-    Get group rank from world rank.
-
-    A function that returns the rank id in the user group corresponding to the
-    rank which id is 'world_rank_id' in the world group.
-
-    Returns:
-        An integer scalar with the rank id in the user group.
-    """
-    if context.get_context("mode") is context.PYNATIVE_MODE:
-        raise RuntimeError("The function 'get_group_rank_from_world_rank' is not supported in PYNATIVE_MODE, "
-                           "'get_group_rank_from_world_rank' only support GRAPH_MODE")
-    check_group(group)
-    check_rank_id(world_rank_id)
-    c_group = c_str(group)
-    c_world_rank_id = ctypes.c_uint(world_rank_id)
-    c_group_rank_id = ctypes.c_uint()
-    ret = HCCL_LIB_CTYPES.HcomGetGroupRankFromWorldRank(c_world_rank_id, c_group, ctypes.byref(c_group_rank_id))
-    if ret != 0:
-        raise RuntimeError('Get group rank from world rank error.')
-
-    return c_group_rank_id.value

mindspore/experimental/llm_boost/ascend_native/llama_boost_ascend_native.py

@@ -1,207 +0,0 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""AscendNative Llama Boost APIs."""
-
-import os
-import numpy as np
-from mindspore.common import Tensor, dtype
-from mindspore.experimental.llm_boost.ascend_native.llm_boost import LLMBoost
-
-def RoundUp(val: int, align: int) -> int:
-    if align == 0:
-        return 0
-    return -(val // -align) * align
-
-
-def ConvertTensor(nd_mat: np.ndarray, transpose: bool = True, nd2nz: bool = True) -> np.ndarray:
-    """ Transforms tensor format from Nd to Nz """
-    if transpose:
-        nd_mat = np.transpose(nd_mat)
-    if not nd2nz:
-        return nd_mat
-    block_size = (16, 16)
-    r = RoundUp(nd_mat.shape[0], block_size[0])
-    c = RoundUp(nd_mat.shape[1], block_size[1])
-    r_pad = r - nd_mat.shape[0]
-    c_pad = c - nd_mat.shape[1]
-    nd_mat = np.pad(nd_mat, ((0, r_pad), (0, c_pad)))
-    nz_mat = np.transpose(np.reshape(
-        nd_mat, (r, c // block_size[1], block_size[1])), (1, 0, 2))
-    nz_mat = nz_mat.reshape(r, c)
-    return nz_mat
-
-class LlamaBoostAscendNative(LLMBoost):
-    r"""
-    Implements an Llama model in a single kernel.
-    it forwards the python functions to the C++ binded object
-    """
-    def _get_from_dict(self, dictionary, name):
-        """ internal function to get a specific tensor from the dictionary """
-        all_relevant_layers = [value for key, value in dictionary.items() if name in key]
-        if all_relevant_layers:
-            return all_relevant_layers[0].asnumpy()
-        return None
-
-    def _get_quant_triplet_from_dict(self, dictionary, name):
-        """ internal function to get a weight triple tensor from the dictionary """
-        weights = self._get_from_dict(dictionary, name + "._handler.weight")
-        scale = self._get_from_dict(dictionary, name + "._weight_quantizer.scale")
-        offset = self._get_from_dict(dictionary, name + "._weight_quantizer.zp_neg")
-        return weights, scale, offset
-
-    def _prepare_single_layer(self, ckpt, config, id):
-        """ prepares the dictionary of weights of a single layer """
-        prefix = 'model.layers.' + str(id)
-        is_last = id == config.num_layers-1
-        layer = 'layers.' + str(id) + '.'
-        l_dict = {key: value for key, value in ckpt.items() if layer in key}
-        if config.n_kv_heads is None:
-            config.n_kv_heads = config.num_heads
-        start = 0
-        end = config.hidden_size
-        kv_start = 0
-        kv_end = int(config.hidden_size*config.n_kv_heads/config.num_heads)
-        ffn_hid = [value for key, value in l_dict.items() if "w3" in key][0].shape[0]
-        ffn_start = 0
-        ffn_end = ffn_hid
-        rank_size = int(os.getenv('RANK_SIZE', '1'))
-        #Emir if (config.parallel_mode != 2): # 2 - AUTO_PARALLEL
-        hid_size = end
-        kv_hid_size = kv_end
-        embed_size = config.vocab_size
-        rank_id = int(os.getenv('RANK_ID', '0'))
-        if (hid_size % rank_size == 0) and (ffn_hid % rank_size == 0) and (embed_size % rank_size == 0):
-            start = int(rank_id * hid_size / rank_size)
-            end = int((rank_id + 1) * hid_size / rank_size)
-            kv_start = int(rank_id * kv_hid_size / rank_size)
-            kv_end = int((rank_id + 1) * kv_hid_size / rank_size)
-            ffn_start = int(rank_id * ffn_hid / rank_size)
-            ffn_end = int((rank_id + 1) * ffn_hid / rank_size)
-        else:
-            raise RuntimeError("hidden size and ffn hidden size must be divided by rank size without remainder.  \
-                                hidden_size: ", hid_size, " ffn_hidden_size: ", ffn_hid, " rank_size: ", rank_size)
-        quant = self._get_from_dict(l_dict, "_weight_quantizer") is not None
-        unite_qkv = config.num_heads == config.n_kv_heads
-        self.dictionary[prefix + ".attention_norm.weight"] = \
-            Tensor(self._get_from_dict(l_dict, "attention_norm"), dtype=dtype.float16)
-        self.dictionary[prefix + ".ffn_norm.weight"] = \
-            Tensor(self._get_from_dict(l_dict, "ffn_norm"), dtype=dtype.float16)
-        if is_last:
-            self.dictionary['lm_head.weight'] = Tensor(ConvertTensor(ckpt['lm_head.weight'].asnumpy()[:, start:end]))
-
-        if not quant:
-            self._pack_attn_weights(l_dict, prefix, start, end, kv_start, kv_end, unite_qkv)
-            self._pack_ffn_weights(l_dict, prefix, ffn_start, ffn_end)
-        else:
-            self._pack_attn_quant_weights(l_dict, prefix, start, end, kv_start, kv_end, unite_qkv)
-            self._pack_ffn_quant_weights(l_dict, prefix, ffn_start, ffn_end)
-
-    def _pack_attn_weights(self, l_dict, prefix, start, end, kv_start, kv_end, unite_qkv):
-        """ prepares the dictionary of weights of an attention block """
-        wq = self._get_from_dict(l_dict, "wq")[start:end, :]
-        wk = self._get_from_dict(l_dict, "wk")[kv_start:kv_end, :]
-        wv = self._get_from_dict(l_dict, "wv")[kv_start:kv_end, :]
-        self.dictionary[prefix + ".attention.wo.weight"] = \
-            Tensor(ConvertTensor(self._get_from_dict(l_dict, "wo")[:, start:end]))
-        if unite_qkv:
-            self.dictionary[prefix + ".attention.wqkv.weight"] = Tensor(ConvertTensor(np.concatenate((wq, wk, wv))))
-        else:
-            self.dictionary[prefix + ".attention.wq.weight"] = Tensor(ConvertTensor(wq))
-            self.dictionary[prefix + ".attention.wkv.weight"] = Tensor(ConvertTensor(np.concatenate((wk, wv))))
-
-    def _pack_ffn_weights(self, l_dict, prefix, ffn_start, ffn_end):
-        """ prepares the dictionary of weights of an ffn block """
-        self.dictionary[prefix + ".feed_forward.w2.weight"] = \
-            Tensor(ConvertTensor(self._get_from_dict(l_dict, "w2")[:, ffn_start:ffn_end]))
-        w1 = self._get_from_dict(l_dict, "w1")[ffn_start:ffn_end, :]
-        w3 = self._get_from_dict(l_dict, "w3")[ffn_start:ffn_end, :]
-        self.dictionary[prefix + ".feed_forward.w13.weight"] = Tensor(ConvertTensor(np.concatenate((w1, w3))))
-
-    def _pack_attn_quant_weights(self, l_dict, prefix, start, end, kv_start, kv_end, unite_qkv):
-        """ prepares the dictionary of weights of a quantized attention block """
-        wq, wq_scale, wq_offset = self._get_quant_triplet_from_dict(l_dict, "wq")
-        wk, wk_scale, wk_offset = self._get_quant_triplet_from_dict(l_dict, "wk")
-        wv, wv_scale, wv_offset = self._get_quant_triplet_from_dict(l_dict, "wv")
-        wo, wo_scale, wo_offset = self._get_quant_triplet_from_dict(l_dict, "wo")
-        self.dictionary[prefix + ".attention.wo.weight"] = Tensor(ConvertTensor(wo[:, start:end], nd2nz=False))
-        self.dictionary[prefix + ".attention.wo.weight.scale"] = Tensor(wo_scale[start:end])
-        self.dictionary[prefix + ".attention.wo.weight.offset"] = Tensor(wo_offset[start:end])
-
-        if unite_qkv:
-            self.dictionary[prefix + ".attention.wqkv.weight"] = \
-             Tensor(ConvertTensor(np.concatenate((wq[start:end, :], wk[kv_start:kv_end, :], wv[kv_start:kv_end, :])),
-                                  nd2nz=False))
-            self.dictionary[prefix + ".attention.wqkv.weight.scale"] = \
-                Tensor(np.concatenate((wq_scale[start:end], wk_scale[kv_start:kv_end], wv_scale[kv_start:kv_end])))
-            self.dictionary[prefix + ".attention.wqkv.weight.offset"] = \
-                Tensor(np.concatenate((wq_offset[start:end], wk_offset[kv_start:kv_end], wv_offset[kv_start:kv_end])))
-        else:
-            self.dictionary[prefix + ".attention.wq.weight"] = Tensor(ConvertTensor(wq[start:end, :], nd2nz=False))
-            self.dictionary[prefix + ".attention.wq.weight.scale"] = Tensor(wq_scale[start:end])
-            self.dictionary[prefix + ".attention.wq.weight.offset"] = Tensor(wq_offset[start:end])
-            self.dictionary[prefix + ".attention.wkv.weight"] = \
-                Tensor(ConvertTensor(np.concatenate((wk[kv_start:kv_end, :], wv[kv_start:kv_end, :])), nd2nz=False))
-            self.dictionary[prefix + ".attention.wkv.weight.scale"] = \
-                Tensor(np.concatenate((wk_scale[kv_start:kv_end], wv_scale[kv_start:kv_end])))
-            self.dictionary[prefix + ".attention.wkv.weight.offset"] = \
-                Tensor(np.concatenate((wk_offset[kv_start:kv_end], wv_offset[kv_start:kv_end])))
-
-    def _pack_ffn_quant_weights(self, l_dict, prefix, ffn_start, ffn_end):
-        """ prepares the dictionary of weights of a quantized ffn block """
-        w1, w1_scale, w1_offset = self._get_quant_triplet_from_dict(l_dict, "w1")
-        w2, w2_scale, w2_offset = self._get_quant_triplet_from_dict(l_dict, "w2")
-        w3, w3_scale, w3_offset = self._get_quant_triplet_from_dict(l_dict, "w3")
-        self.dictionary[prefix + ".feed_forward.w2.weight"] = Tensor(ConvertTensor(w2[:, ffn_start:ffn_end],
-                                                                                   nd2nz=False))
-        self.dictionary[prefix + ".feed_forward.w2.weight.scale"] = Tensor(w2_scale[ffn_start:ffn_end])
-        self.dictionary[prefix + ".feed_forward.w2.weight.offset"] = Tensor(w2_offset[ffn_start:ffn_end])
-
-        self.dictionary[prefix + ".feed_forward.w13.weight"] = \
-                Tensor(ConvertTensor(np.concatenate((w1[ffn_start:ffn_end, :], w3[ffn_start:ffn_end, :])), nd2nz=False))
-        self.dictionary[prefix + ".feed_forward.w13.weight.scale"] = \
-                Tensor(np.concatenate((w1_scale[ffn_start:ffn_end], w3_scale[ffn_start:ffn_end])))
-        self.dictionary[prefix + ".feed_forward.w13.weight.offset"] = \
-            Tensor(np.concatenate((w1_offset[ffn_start:ffn_end], w3_offset[ffn_start:ffn_end])))
-
-    def _prepare_cos_sin_arrays(self, config, theta=10000):
-        """ prepares the cosine and sine arrays """
-        head_dim = config.hidden_size // config.num_heads
-        max_position_embedding = \
-            config.max_position_embedding if config.max_position_embedding is not None else config.seq_length
-        freqs_base = np.arange(0, head_dim, 2)[: (head_dim // 2)].astype(np.float32)
-        freqs = 1.0 / (theta ** (freqs_base / head_dim))
-        t = np.arange(0, max_position_embedding, 1).astype(np.float32)
-        freqs = np.outer(t, freqs)
-        emb = np.concatenate((freqs, freqs), axis=-1)
-        freqs_cos = Tensor(np.cos(emb), dtype=dtype.float16)
-        sin = np.sin(emb)
-
-        sin[:, :int(emb.shape[1]/2)] = -sin[:, :int(emb.shape[1]/2)]
-        self.dictionary['model.cos.weight'] = freqs_cos
-        freqs_sin = Tensor(sin, dtype=dtype.float16)
-        self.dictionary['model.sin.weight'] = freqs_sin
-
-    def set_weights(self, ckpt_dict):
-        """ load the checkpoint """
-        self.dictionary = {}
-        self.dictionary['model.tok_embeddings.embedding_weight'] = \
-            Tensor(ckpt_dict['model.tok_embeddings.embedding_weight'].asnumpy())
-        self.dictionary['model.norm_out.weight'] = \
-            Tensor(ckpt_dict['model.norm_out.weight'].asnumpy(), dtype=dtype.float16)
-        self._prepare_cos_sin_arrays(self.config)
-        for layer_id in range(self.config.num_layers):
-            self._prepare_single_layer(ckpt_dict, self.config, layer_id)
-
-        self.binder.set_weights_map(self.dictionary)

mindspore/experimental/llm_boost/ascend_native/llm_boost.py

@@ -1,52 +0,0 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""LLMBoost APIs."""
-
-from mindspore.common import Tensor
-
-class LLMBoost():
-    r"""
-    Implements an LLM in a single kernel.
-    it forwards the python function to the C++ binded object
-    """
-    def __init__(self, config):
-        r"""
-        initialize the parameters of the llm binder.
-        config is simply the config object of the model
-        """
-        from mindspore._c_expression import LlmBoostBinder
-        self.config = config
-        self.binder = LlmBoostBinder("AscendNative", config.model_type)
-        self.binder.init_model(config.to_dict())
-
-    def init(self):
-        """
-        Initialize the object
-        returns True if object needs input manipulation by mindformers
-        """
-        return False
-
-    def set_kvcache(self, k_caches=None, v_caches=None):
-        return
-
-    def forward(self, input_ids, batch_valid_length, position_ids=None):
-        ret = self.binder.forward([input_ids, batch_valid_length], "nothing really")
-        return Tensor(ret[0])
-
-    def set_weights(self, ckpt_dict):
-        self.binder.set_weights_map(ckpt_dict)
-
-    def add_flags(self, is_first_iteration=False):
-        self.binder.add_flags(is_first_iteration=is_first_iteration)

mindspore/experimental/llm_boost/atb/__init__.py

@@ -1,23 +0,0 @@
-# Copyright 2024 Huawei Technologies Co., Ltd
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ============================================================================
-"""
-Provide llm boost for inference, such as LlamaBoost.
-"""
-from __future__ import absolute_import
-
-from mindspore.experimental.llm_boost.atb.llama_boost import LlamaBoost
-from mindspore.experimental.llm_boost.atb.qwen_boost import QwenBoost
-
-__all__ = ['LlamaBoost', 'QwenBoost']