mindspore 2.4.10__cp39-cp39-manylinux1_x86_64.whl → 2.5.0__cp39-cp39-manylinux1_x86_64.whl

mindspore/lib/plugin/ascend/ms_kernels_internal/internal_kernel/include/{acme/include/base_type.h → base_type.h}

@@ -14,16 +14,17 @@
  * limitations under the License.
  */
 
-#ifndef MS_KERNELS_INTERNAL_KERNEL_ACME_BASE_TYPE_H_
-#define MS_KERNELS_INTERNAL_KERNEL_ACME_BASE_TYPE_H_
+#ifndef MS_KERNELS_INTERNAL_KERNEL_BASE_TYPE_H_
+#define MS_KERNELS_INTERNAL_KERNEL_BASE_TYPE_H_
 
 #include <vector>
+#include <unordered_set>
 #include <map>
 #include <cstdint>
 #include <memory>
 
 namespace mindspore {
-namespace acme {
+namespace internal {
 using ShapeInfo = std::vector<int64_t>;
 
 enum DataType : int {
@@ -63,9 +64,9 @@ enum TensorFormat : int {
   kFormatFRACTAL_Z_3D
 };
 
-enum AcmeStatus {
-  kAcmeOk = 0,
-  kAcmeError,
+enum InternalStatus {
+  kInternalOk = 0,
+  kInternalError,
 };
 
 class ArgImmutableInfo {
@@ -87,6 +88,19 @@ class ArgImmutableInfo {
   TensorFormat format_{TensorFormat::kFormatUnknown};
 };
 
+inline size_t CountNumFromShape(const ShapeInfo &shape) {
+  if (shape.empty()) {
+    return 0;
+  }
+
+  size_t num = 1;
+  for (auto s : shape) {
+    num *= static_cast<size_t>(s);
+  }
+
+  return num;
+}
+
 class ArgDesc {
  public:
   ArgDesc(const ArgImmutableInfo &arg_ii) : immutable_info_(arg_ii) {}
@@ -114,18 +128,7 @@ class ArgDesc {
 
   const ArgImmutableInfo &GetImmutableInfo() const { return immutable_info_; }
 
-  inline size_t ElementNum() const {
-    if (shape_.empty()) {
-      return 0;
-    }
-
-    size_t num = 1;
-    for (auto s : shape_) {
-      num *= static_cast<size_t>(s);
-    }
-
-    return num;
-  }
+  inline size_t ElementNum() const { return CountNumFromShape(shape_); }
 
  private:
   ShapeInfo shape_{0};
@@ -148,7 +151,9 @@ using ShapeInfoList = std::vector<ShapeInfo>;
 
 using InOutDtypesList = std::vector<std::vector<InputDataTypes>>;
 using InOutDtypesTargetMap = std::map<std::string, InOutDtypesList>;
-}  // namespace acme
+
+using InOutIndicesType = std::pair<std::unordered_set<size_t>, std::unordered_set<size_t>>;
+}  // namespace internal
 }  // namespace mindspore
 
-#endif  // MS_KERNELS_INTERNAL_KERNEL_ACME_BASE_TYPE_H_
+#endif  // MS_KERNELS_INTERNAL_KERNEL_BASE_TYPE_H_

mindspore/lib/plugin/ascend/ms_kernels_internal/internal_kernel/include/{cast/cast_tiling.h → internal.h}

@@ -14,9 +14,11 @@
  * limitations under the License.
  */
 
-#ifndef MS_KERNELS_INTERNAL_KERNEL_ASCENDC_CAST_TILING_H_
-#define MS_KERNELS_INTERNAL_KERNEL_ASCENDC_CAST_TILING_H_
+#ifndef MS_KERNELS_INTERNAL_KERNEL_INTERNAL_H_
+#define MS_KERNELS_INTERNAL_KERNEL_INTERNAL_H_
 
-#include "acme/src/ops/device_src/ascendc/cast/cast_tiling.h"
+#include "include/op_param.h"
+#include "include/op_creator.h"
+#include "include/tiling_info.h"
 
-#endif
+#endif  // MS_KERNELS_INTERNAL_KERNEL_INTERNAL_H_

mindspore/lib/plugin/ascend/ms_kernels_internal/internal_kernel/include/internal_op.h

@@ -0,0 +1,114 @@
+/**
+ * Copyright 2024 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MS_KERNELS_INTERNAL_KERNEL_INTERNAL_OP_H_
+#define MS_KERNELS_INTERNAL_KERNEL_INTERNAL_OP_H_
+
+#include <vector>
+#include <string>
+
+#include "include/base_type.h"
+#include "include/op_param.h"
+#include "include/tiling_info.h"
+
+namespace mindspore {
+namespace internal {
+enum OpType : int {
+  kOpTypeAICore = 0,
+  kOpTypeAICpu,
+  kOpTypeAIV,
+  kOpTypeWriteBack,
+  kOpTypeMixAIC,
+  kOpTypeMixAIV,
+  kOpTypeFFTSPlus,
+  kOpTypeDSA,
+  kOpTypeDVPP,
+  kOpTypeHCCL,
+  kOpTypeInvalid
+};
+
+class InternalOp {
+ public:
+  InternalOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+         const std::string &op_name);
+  virtual ~InternalOp() = default;
+  InternalStatus Init();
+
+  virtual InternalStatus UpdateShape(const ShapeInfoList &inputs_shape, const ShapeInfoList &outputs_shape);
+  virtual InternalStatus UpdateParam(const void *) { return kInternalOk; }
+
+  virtual size_t GetTilingSize() const;
+  virtual std::vector<size_t> GetWorkspaceSize() const;
+
+  virtual void SetTilingInfo(const TilingInfoPtr &tiling_info);
+
+  virtual InternalStatus Launch(const InputsAddrList &input_ptrs, const OutputsAddrList &output_ptrs, const WsAddrList &ws_ptrs,
+                    void *stream, const std::string &op_fullname = "");
+  virtual InternalStatus Tiling(RawHostAddr host_ptr, HostRunInfoPtr *run_info_ptr);
+  virtual std::string DumpTiling(const RawHostAddr host_ptr) const = 0;
+
+  virtual ShapeInfoList InferShape(const ShapeInfoList &inputs_shape) const = 0;
+
+  virtual InternalStatus TilingFromTuning(const RawDeviceAddr tiling_addr);
+  virtual bool IsSupported(const InputDataTypes &dtypes);
+
+  virtual std::string GetOpName() { return "Internal" + op_name_; };
+  std::string GetOpNameOrigin() { return op_name_; };
+  virtual uint32_t GetLaunchCoreNum() const { return host_run_info_comm_ptr_->block_dim_; };
+  virtual OpType GetOpType() = 0;
+
+  // ---- Ops DataBase related functions ----
+  // Version: Op implementation version(update when the unnegligible change happens)
+  virtual int64_t GetOpVersion() const { return 0; }
+  // DataBaseKey: Op information(which can rebuild this op without ambiguity)
+  virtual std::vector<int64_t> GetDataBaseKey() const { return std::vector<int64_t>{}; }
+  // Tiling
+  virtual std::vector<int64_t> GetCurrentTiling() const { return {}; }
+  virtual InternalStatus Tiling(const std::vector<int64_t> &tiling, RawHostAddr &tiling_addr,
+                            HostRunInfoPtr *run_info_ptr) {
+    return kInternalError;
+  }
+  // ---- Ops DataBase related functions ----
+
+ protected:
+  virtual InternalStatus InitImpl();
+  virtual InternalStatus TilingImpl(RawHostAddr host_ptr, HostRunInfoPtr *run_info_ptr) = 0;
+  virtual InternalStatus LaunchImpl(const InputsAddrList &input_ptrs, const OutputsAddrList &output_ptrs,
+                                const WsAddrList &ws_ptrs, void *stream) = 0;
+  void SetHostRunInfoComm(const HostRunInfoComm &, HostRunInfoPtr *);
+
+  InputsImmutableInfoList inputs_ii_;
+  OutputsImmutableInfoList outputs_ii_;
+  ShapeInfoList inputs_shape_;
+  ShapeInfoList outputs_shape_;
+  std::string op_name_{"UnknownOp"};
+  size_t tiling_size_{0};
+  std::vector<size_t> ws_size_;
+  RawDeviceAddr tiling_device_addr_{nullptr};
+  HostRunInfoCommPtr host_run_info_comm_ptr_{nullptr};
+
+ private:
+  virtual InternalStatus LaunchWithProfiling(const InputsAddrList &input_ptrs, const OutputsAddrList &output_ptrs,
+                                         const WsAddrList &ws_ptrs, void *stream, const std::string &op_fullname);
+};
+
+using InternalOpPtr = std::shared_ptr<InternalOp>;
+
+short GetPlatformEnum(const std::string &soc_name);
+}  // namespace internal
+}  // namespace mindspore
+
+#endif  // MS_KERNELS_INTERNAL_KERNEL_INTERNAL_OP_H_

mindspore/lib/plugin/ascend/ms_kernels_internal/internal_kernel/include/llm/boost_kernel.h

@@ -0,0 +1,70 @@
+/**
+ * Copyright 2024 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MS_KERNELS_INTERNAL_KERNEL_INCLUDE_LLM_BOOST_KERNEL_H_
+#define MS_KERNELS_INTERNAL_KERNEL_INCLUDE_LLM_BOOST_KERNEL_H_
+#include <memory>
+#include <vector>
+#include "include/llm/tensor.h"
+
+namespace mindspore {
+namespace internal {
+
+struct BoostParam {};
+struct RawBuf {
+  uint64_t size_{0};
+  void *addr_{nullptr};
+};
+using HostRawBuf = RawBuf;
+using DeviceRawBuf = RawBuf;
+
+using BoostParamPtr = std::shared_ptr<BoostParam>;
+
+class BoostKernel {
+ public:
+  BoostKernel(const BoostParamPtr &param) : param_(param){};
+  virtual ~BoostKernel() {}
+  // this routine will check if this kernel can support the requirements
+  // specified in ValidationInfo.
+  virtual bool Init() = 0;
+  virtual void SetInputs(const std::vector<Tensor *> &inputs) { inputs_ = inputs; }
+  virtual void SetOutputs(const std::vector<Tensor *> &outputs) { outputs_ = outputs; }
+  virtual void SetWorkSpace(const std::vector<DeviceRawBuf> &workspace) { return; }
+  virtual void SetStream(const void *stream_ptr) { stream_ptr_ = const_cast<void *>(stream_ptr); }
+  virtual void SetDeviceTilingBuf(const DeviceRawBuf &tilingBuf) = 0;
+  virtual int Launch() = 0;
+  virtual uint64_t GetTilingBufSize() = 0;
+  virtual int Tiling(HostRawBuf &tilingBuf) = 0;
+  virtual std::vector<uint64_t> GetWorkSpaceSize() = 0;
+  virtual int InferShape(const std::vector<TensorDesc> &inputs, std::vector<TensorDesc> &outputs) = 0;
+  virtual bool IsSupported() { return true; }
+
+  virtual std::vector<Tensor *> &get_inputs() { return inputs_; }
+  virtual std::vector<Tensor *> &get_outputs() { return outputs_; }
+
+  std::string get_name() { return kernel_name_; }
+  void set_name(std::string kernel_name) { kernel_name_ = kernel_name; }
+
+ protected:
+  std::string kernel_name_;
+  BoostParamPtr param_ = nullptr;
+  std::vector<Tensor *> inputs_;
+  std::vector<Tensor *> outputs_;
+  void *stream_ptr_ = nullptr;
+};
+using BoostKernelPtr = std::shared_ptr<BoostKernel>;
+}  // namespace internal
+}  // namespace mindspore
+#endif  // MS_KERNELS_INTERNAL_KERNEL_INCLUDE_LLM_BOOST_KERNEL_H_

mindspore/lib/plugin/ascend/ms_kernels_internal/internal_kernel/include/llm/llama_impl.h

@@ -0,0 +1,85 @@
+/**
+ * Copyright 2024 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MS_KERNELS_INTERNAL_KERNEL_INCLUDE_LLM_LLAMA_IMPL_H_
+#define MS_KERNELS_INTERNAL_KERNEL_INCLUDE_LLM_LLAMA_IMPL_H_
+
+#include <iostream>
+#include <memory>
+#include <string>
+#include <vector>
+#include "include/llm/boost_kernel.h"
+#include "include/llm/model_interface.h"
+
+namespace mindspore {
+namespace internal {
+struct OpLlamaModelParam : public BoostParam {
+  int batch_size_;
+  int seq_length_;
+  int head_num_;
+  int kv_head_num_;
+  int hidden_size_;
+  int num_layers_;
+  float ln_eps_;
+  int vocab_size_;
+  int multiple_of_;
+  int device_id_;
+  int device_num_;
+  bool paged_attention_ = false;
+  int64_t page_size_;
+  int page_num_;
+  int table_id_;
+  void *hcom_ = nullptr;
+};
+
+using OpLlamaModelParamPtr = std::shared_ptr<OpLlamaModelParam>;
+
+class LlamaImpl : public BoostKernel, public ModelInterface {
+ public:
+  explicit LlamaImpl(const OpLlamaModelParamPtr param);
+  virtual ~LlamaImpl();
+  bool Init() override;
+  void SetDeviceTilingBuf(const DeviceRawBuf &tilingBuf) override;
+  int Launch() override;
+  uint64_t GetTilingBufSize() override;
+  int Tiling(HostRawBuf &tilingBuf) override;
+  int InferShape(const std::vector<TensorDesc> &inputs, std::vector<TensorDesc> &outputs) override;
+  void PrintModel();
+  static int CreateDictFromCKPT(dict *dict, std::string name);
+  int AllocTable() override;
+  int FreeTable(int table_id) override;
+  bool AclInit();
+
+ private:
+  bool HcclInit();
+  bool AllocKVCacheTable();
+  bool BuildLlamaModel();
+  bool LlamaAllocateInnerTensors();
+  void *AllocateWs(size_t ws_size) override;
+  virtual std::vector<uint64_t> GetWorkSpaceSize() override;
+  bool SetupWorkspace() override;
+  void SetWorkSpace(const std::vector<DeviceRawBuf> &workspace) override;
+  int HandleDynamicInput();
+  int SetupInputTensors();
+
+  DeviceRawBuf tiling_buf_;
+
+  void *prefil_mode_dev_ = nullptr;
+  void *decode_mode_dev_ = nullptr;
+  size_t batch_ = 0;
+};
+}  // namespace internal
+}  // namespace mindspore
+#endif  // MS_KERNELS_INTERNAL_KERNEL_INCLUDE_LLM_LLAMA_IMPL_H_

mindspore/lib/plugin/ascend/ms_kernels_internal/internal_kernel/include/llm/model_interface.h

@@ -0,0 +1,52 @@
+/**
+ * Copyright 2024 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef MS_KERNELS_INTERNAL_KERNEL_INCLUDE_LLM_MODEL_INTERFACE_H_
+#define MS_KERNELS_INTERNAL_KERNEL_INCLUDE_LLM_MODEL_INTERFACE_H_
+
+#include <map>
+#include "include/llm/tensor.h"
+
+namespace mindspore {
+namespace internal {
+
+using dict = std::map<std::string, Tensor *>;
+class CacheMgr;
+class Graph;
+
+class ModelInterface {
+ public:
+  ModelInterface(int dev_id, int num_layers, int seq_len, int page_num, int page_size, int head_num,
+                 int kv_head_num, int hidden_size);
+  virtual ~ModelInterface();
+  virtual int AllocTable() = 0;
+  virtual int FreeTable(int table_id) = 0;
+  virtual void *AllocateWs(size_t ws_size) = 0;
+  virtual bool SetupWorkspace() = 0;
+  void SetIsFIrstIter(bool is_first_iter) { is_first_iter_ = is_first_iter; }
+  void SetupWeights(dict *dict_weights) { dict_weights_ = dict_weights; }
+
+ protected:
+  Graph *graph_ = nullptr;
+  CacheMgr *cache_mgr_;
+  dict *dict_weights_ = nullptr;
+  bool is_first_iter_ = true;
+  void *workspace_addr_ = nullptr;
+  size_t workspace_size_ = 0;
+  static constexpr size_t max_ws_size_ = static_cast<size_t>(8000) * (1 << 20);
+};
+}  // namespace internal
+}  // namespace mindspore
+#endif  // MS_KERNELS_INTERNAL_KERNEL_INCLUDE_LLM_MODEL_INTERFACE_H_

mindspore/lib/plugin/ascend/ms_kernels_internal/internal_kernel/include/llm/tensor.h

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MS_KERNELS_INTERNAL_KERNEL_SRC_OPS_LLAMA_TENSOR_H_
+#define MS_KERNELS_INTERNAL_KERNEL_SRC_OPS_LLAMA_TENSOR_H_
+#include <vector>
+#include <map>
+#include <limits>
+#include <iostream>
+#include "include/base_type.h"
+
+using DIMS = std::vector<int64_t>;
+namespace mindspore::internal {
+constexpr size_t HALF_DATA_SIZE = 2;
+enum TensorDType : int {
+  TENSOR_DTYPE_UNDEFINED = -1,
+  TENSOR_DTYPE_FLOAT = 0,
+  TENSOR_DTYPE_FLOAT16 = 1,
+  TENSOR_DTYPE_INT8 = 2,
+  TENSOR_DTYPE_INT32 = 3,
+  TENSOR_DTYPE_UINT8 = 4,
+  TENSOR_DTYPE_INT16 = 6,
+  TENSOR_DTYPE_UINT16 = 7,
+  TENSOR_DTYPE_UINT32 = 8,
+  TENSOR_DTYPE_INT64 = 9,
+  TENSOR_DTYPE_UINT64 = 10,
+  TENSOR_DTYPE_DOUBLE = 11,
+  TENSOR_DTYPE_BOOL = 12,
+  TENSOR_DTYPE_STRING = 13,
+  TENSOR_DTYPE_COMPLEX64 = 16,
+  TENSOR_DTYPE_COMPLEX128 = 17,
+  TENSOR_DTYPE_BF16 = 27
+};
+
+static const std::map<TensorDType, size_t> MAP_OF_DTYPE_SIZE = {{TensorDType::TENSOR_DTYPE_UNDEFINED, 0},
+                                                                {TensorDType::TENSOR_DTYPE_FLOAT, sizeof(float)},
+                                                                {TensorDType::TENSOR_DTYPE_FLOAT16, HALF_DATA_SIZE},
+                                                                {TensorDType::TENSOR_DTYPE_INT8, sizeof(int8_t)},
+                                                                {TensorDType::TENSOR_DTYPE_INT32, sizeof(int32_t)},
+                                                                {TensorDType::TENSOR_DTYPE_UINT8, sizeof(uint8_t)},
+                                                                {TensorDType::TENSOR_DTYPE_INT16, sizeof(int16_t)},
+                                                                {TensorDType::TENSOR_DTYPE_UINT16, sizeof(uint16_t)},
+                                                                {TensorDType::TENSOR_DTYPE_UINT32, sizeof(uint32_t)},
+                                                                {TensorDType::TENSOR_DTYPE_INT64, sizeof(int64_t)},
+                                                                {TensorDType::TENSOR_DTYPE_UINT64, sizeof(uint64_t)},
+                                                                {TensorDType::TENSOR_DTYPE_DOUBLE, sizeof(double)},
+                                                                {TensorDType::TENSOR_DTYPE_BOOL, sizeof(bool)},
+                                                                {TensorDType::TENSOR_DTYPE_BF16, HALF_DATA_SIZE}};
+
+struct TensorDesc {
+  TensorDType dtype = TENSOR_DTYPE_UNDEFINED;
+  TensorFormat format = TensorFormat::kFormatUnknown;
+  DIMS dims;
+  int64_t Numel() const;
+};
+
+struct Tensor {
+  TensorDesc desc;
+  void *data = nullptr;
+  void *hostData = nullptr;
+  size_t dataSize = 0;
+  int64_t Numel() const;
+};
+
+size_t GetTensorElementSize(const TensorDType dtype);
+
+}  // namespace mindspore::internal
+#endif  // MS_KERNELS_INTERNAL_KERNEL_SRC_OPS_LLAMA_TENSOR_H_

mindspore/lib/plugin/ascend/ms_kernels_internal/internal_kernel/include/op_creator.h

@@ -0,0 +1,123 @@
+/**
+ * Copyright 2024 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MS_KERNELS_INTERNAL_KERNEL_OP_CREATOR_H_
+#define MS_KERNELS_INTERNAL_KERNEL_OP_CREATOR_H_
+
+#include "include/internal_op.h"
+#include "include/op_param.h"
+
+namespace mindspore {
+namespace internal {
+InternalOpPtr CreateMatmulOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                         const MatmulParam &param, const std::string &op_name);
+InternalOpPtr CreateAddOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                      const std::string &op_name);
+InternalOpPtr CreateAddLayerNormOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                               const NormParam &param, const std::string &op_name);
+InternalOpPtr CreateAddRmsNormDynamicQuantOp(const InputsImmutableInfoList &inputs_ii,
+                                         const OutputsImmutableInfoList &outputs_ii, const NormParam &param,
+                                         const std::string &op_name);
+InternalOpPtr CreateAddRmsNormQuantOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                                  const NormParam &param, const std::string &op_name);
+InternalOpPtr CreateCastOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                       const std::string &op_name);
+InternalOpPtr CreateTransposeOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                            const TransposeParam &param, const std::string &op_name);
+InternalOpPtr CreateQuantPerChannelOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                                  const std::string &op_name);
+InternalOpPtr CreateSwishOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                        const std::string &op_name);
+InternalOpPtr CreateSwiGLUOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                         const SwiGLUParam &param, const std::string &op_name);
+InternalOpPtr CreateLogicalNotOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                             const std::string &op_name);
+InternalOpPtr CreateSoftmaxOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                          const SoftmaxParam &param, const std::string &op_name);
+InternalOpPtr CreateReduceSumOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                            const ReduceSumParam &param, const std::string &op_name);
+InternalOpPtr CreateGatherOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                         const GatherParam &param, const std::string &op_name);
+InternalOpPtr CreateApplyRotaryPosEmbOp(const InputsImmutableInfoList &inputs_ii,
+                                    const OutputsImmutableInfoList &outputs_ii, const ApplyRotaryPosEmbParam &param,
+                                    const std::string &op_name);
+InternalOpPtr CreateApplyRotaryPosEmbNzOp(const InputsImmutableInfoList &inputs_ii,
+                                      const OutputsImmutableInfoList &outputs_ii, const ApplyRotaryPosEmbParam &param,
+                                      const std::string &op_name);
+InternalOpPtr CreateRmsNormOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                          const NormParam &param, const std::string &op_name);
+InternalOpPtr CreateMatmulAddRmsNormOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                                   const MatmulAddRmsNormParam &param, const std::string &op_name);
+InternalOpPtr CreateMultiWeightMatmulOp(const InputsImmutableInfoList &inputs_ii,
+                                    const OutputsImmutableInfoList &outputs_ii, const MultiWeightMatmulParam &param,
+                                    const std::string &op_name);
+// param section 0
+InternalOpPtr CreateGeLUOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                       const std::string &op_name);
+InternalOpPtr CreateAddRmsNormOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                             const NormParam &param, const std::string &op_name);
+InternalOpPtr CreateFlashAttentionScoreOp(const InputsImmutableInfoList &inputs_ii,
+                                      const OutputsImmutableInfoList &outputs_ii, const FlashAttentionScoreParam &param,
+                                      const std::string &op_name);
+InternalOpPtr CreatePagedAttentionOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                                 const PagedAttentionParam &param, const std::string &op_name);
+// param section 1
+InternalOpPtr CreateReshapeAndCacheOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                                  const std::string &op_name);
+InternalOpPtr CreateReshapeAndCacheNzOp(const InputsImmutableInfoList &inputs_ii,
+                                    const OutputsImmutableInfoList &outputs_ii, const std::string &op_name);
+InternalOpPtr CreateMulOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                      const std::string &op_name);
+InternalOpPtr CreateSubOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                      const std::string &op_name);
+InternalOpPtr CreateRealDivOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                          const std::string &op_name);
+
+// param section 2
+InternalOpPtr CreateFastGeLUOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                           const std::string &op_name);
+InternalOpPtr CreateTransDataOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                            const TransDataParam &param, const std::string &op_name);
+InternalOpPtr CreateQuantLinearSparseOp(const InputsImmutableInfoList &inputs_ii,
+                                    const OutputsImmutableInfoList &outputs_ii, const std::string &op_name);
+InternalOpPtr CreateTssAddLayerNormOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                                  const std::string &op_name);
+// param section 3
+InternalOpPtr CreateLessOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                       const std::string &op_name);
+InternalOpPtr CreateEqualOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                        const std::string &op_name);
+InternalOpPtr CreateNotEqualOp(const InputsImmutableInfoList &inputs_ii, const OutputsImmutableInfoList &outputs_ii,
+                           const std::string &op_name);
+// param section 4
+
+// param section 5
+
+// param section 6
+
+// param section 7
+
+// param section 8
+
+// param section 9
+
+// param section 10
+
+bool IsInternalKernelDtypesSupported(const std::string op_name, InputDataTypes in_dtypes, InputDataTypes out_dtypes);
+}  // namespace internal
+}  // namespace mindspore
+
+#endif  // MS_KERNELS_INTERNAL_KERNEL_OP_CREATOR_H_