mindspore 2.2.0__cp38-cp38-manylinux1_x86_64.whl → 2.2.10__cp38-cp38-manylinux1_x86_64.whl

mindspore/include/api/model.h

@@ -136,6 +136,13 @@ class MS_API Model {
   /// \return Status.
   Status UpdateWeights(const std::vector<MSTensor> &new_weights);
 
+  /// \brief Change the size and or content of weight tensors
+  ///
+  /// \param[in]  A vector where model constant are arranged in sequence
+  ///
+  /// \return Status.
+  Status UpdateWeights(const std::vector<std::vector<MSTensor>> &new_weights);
+
   /// \brief Inference model API. If use this API in train mode, it's equal to RunStep API.
   ///
   /// \param[in] inputs A vector where model inputs are arranged in sequence.
@@ -358,6 +365,13 @@ class MS_API Model {
 
   const std::shared_ptr<ModelImpl> impl() const { return impl_; }
 
+  /// \brief Get model info by key
+  ///
+  /// \param[in] key The key of model info key-value pair
+  ///
+  /// \return The value of the model info associated with the given key.
+  inline std::string GetModelInfo(const std::string &key);
+
  private:
   friend class Serialization;
   // api without std::string
@@ -374,6 +388,7 @@ class MS_API Model {
                const std::vector<char> &cropto_lib_path);
   Status Build(const std::vector<char> &model_path, ModelType model_type, const std::shared_ptr<Context> &model_context,
                const Key &dec_key, const std::vector<char> &dec_mode, const std::vector<char> &cropto_lib_path);
+  std::vector<char> GetModelInfo(const std::vector<char> &key);
   std::shared_ptr<ModelImpl> impl_;
 };
 
@@ -416,5 +431,7 @@ Status Model::Build(const std::string &model_path, ModelType model_type,
                     const std::shared_ptr<Context> &model_context) {
   return Build(StringToChar(model_path), model_type, model_context);
 }
+
+inline std::string Model::GetModelInfo(const std::string &key) { return CharToString(GetModelInfo(StringToChar(key))); }
 }  // namespace mindspore
 #endif  // MINDSPORE_INCLUDE_API_MODEL_H

mindspore/include/api/status.h

@@ -83,9 +83,14 @@ enum StatusCode : uint32_t {
   kLiteModelRebuild = kLite | (0x0FFFFFFF & -12),    /**< Model has been built. */
 
   // Executor error code, range: [-100,-200)
-  kLiteOutOfTensorRange = kLite | (0x0FFFFFFF & -100), /**< Failed to check range. */
-  kLiteInputTensorError = kLite | (0x0FFFFFFF & -101), /**< Failed to check input tensor. */
-  kLiteReentrantError = kLite | (0x0FFFFFFF & -102),   /**< Exist executor running. */
+  kLiteOutOfTensorRange = kLite | (0x0FFFFFFF & -100),           /**< Failed to check range. */
+  kLiteInputTensorError = kLite | (0x0FFFFFFF & -101),           /**< Failed to check input tensor. */
+  kLiteReentrantError = kLite | (0x0FFFFFFF & -102),             /**< Exist executor running. */
+  kLiteLLMWaitProcessTimeOut = kLite | (0x0FFFFFFF & -103),      /**< Wait to be processed time out. */
+  kLiteLLMKVCacheNotExist = kLite | (0x0FFFFFFF & -104),         /**< KV Cache not exist. */
+  kLiteLLMRepeatRequest = kLite | (0x0FFFFFFF & -105),           /**< repeat request. */
+  kLiteLLMRequestAlreadyCompleted = kLite | (0x0FFFFFFF & -106), /**< request already complete!. */
+  kLiteLLMEngineFinalized = kLite | (0x0FFFFFFF & -107),         /**< llm engine finalized. */
 
   // Graph error code, range: [-200,-300)
   kLiteGraphFileError = kLite | (0x0FFFFFFF & -200), /**< Failed to verify graph file. */

mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend310/aic-ascend310-ops-info.json

@@ -0,0 +1,123 @@
+{
+    "AddDSL":{
+        "input0":{
+            "dtype":"float16",
+            "format":"NCHW",
+            "name":"x1",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float16",
+            "format":"NCHW",
+            "name":"x2",
+            "paramType":"required"
+        },
+        "opFile":{
+            "value":"add_dsl"
+        },
+        "opInterface":{
+            "value":"add_dsl"
+        },
+        "output0":{
+            "dtype":"float16",
+            "format":"NCHW",
+            "name":"y",
+            "paramType":"required"
+        }
+    },
+    "AddTik":{
+        "input0":{
+            "dtype":"float16",
+            "format":"ND",
+            "name":"x1",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float16",
+            "format":"ND",
+            "name":"x2",
+            "paramType":"required"
+        },
+        "opFile":{
+            "value":"add_tik"
+        },
+        "opInterface":{
+            "value":"add_tik"
+        },
+        "output0":{
+            "dtype":"float16",
+            "format":"ND",
+            "name":"y",
+            "paramType":"required"
+        }
+    },
+    "FlashAttention":{
+        "input0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"q",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"k",
+            "paramType":"required"
+        },
+        "input2":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"v",
+            "paramType":"required"
+        },
+        "input3":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"attention_mask",
+            "paramType":"required"
+        },
+        "opFile":{
+            "value":"flash_attention"
+        },
+        "opInterface":{
+            "value":"flash_attention"
+        },
+        "output0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"y",
+            "paramType":"required"
+        }
+    },
+    "MatmulTik":{
+        "input0":{
+            "dtype":"int8,uint8,float16",
+            "format":"ND,ND,ND",
+            "name":"x1",
+            "needCompile":"false",
+            "paramType":"required",
+            "shape":"all"
+        },
+        "input1":{
+            "dtype":"int8,int8,float16",
+            "format":"ND,ND,ND",
+            "name":"x2",
+            "needCompile":"false",
+            "paramType":"required",
+            "shape":"all"
+        },
+        "opFile":{
+            "value":"matmul_tik"
+        },
+        "opInterface":{
+            "value":"matmul_tik"
+        },
+        "output0":{
+            "dtype":"int32,int32,float",
+            "format":"ND,ND,ND",
+            "name":"y",
+            "paramType":"required",
+            "shape":"all"
+        }
+    }
+}

mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend310p/aic-ascend310p-ops-info.json

@@ -0,0 +1,123 @@
+{
+    "AddDSL":{
+        "input0":{
+            "dtype":"float16",
+            "format":"NCHW",
+            "name":"x1",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float16",
+            "format":"NCHW",
+            "name":"x2",
+            "paramType":"required"
+        },
+        "opFile":{
+            "value":"add_dsl"
+        },
+        "opInterface":{
+            "value":"add_dsl"
+        },
+        "output0":{
+            "dtype":"float16",
+            "format":"NCHW",
+            "name":"y",
+            "paramType":"required"
+        }
+    },
+    "AddTik":{
+        "input0":{
+            "dtype":"float16",
+            "format":"ND",
+            "name":"x1",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float16",
+            "format":"ND",
+            "name":"x2",
+            "paramType":"required"
+        },
+        "opFile":{
+            "value":"add_tik"
+        },
+        "opInterface":{
+            "value":"add_tik"
+        },
+        "output0":{
+            "dtype":"float16",
+            "format":"ND",
+            "name":"y",
+            "paramType":"required"
+        }
+    },
+    "FlashAttention":{
+        "input0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"q",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"k",
+            "paramType":"required"
+        },
+        "input2":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"v",
+            "paramType":"required"
+        },
+        "input3":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"attention_mask",
+            "paramType":"required"
+        },
+        "opFile":{
+            "value":"flash_attention"
+        },
+        "opInterface":{
+            "value":"flash_attention"
+        },
+        "output0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"y",
+            "paramType":"required"
+        }
+    },
+    "MatmulTik":{
+        "input0":{
+            "dtype":"int8,uint8,float16",
+            "format":"ND,ND,ND",
+            "name":"x1",
+            "needCompile":"false",
+            "paramType":"required",
+            "shape":"all"
+        },
+        "input1":{
+            "dtype":"int8,int8,float16",
+            "format":"ND,ND,ND",
+            "name":"x2",
+            "needCompile":"false",
+            "paramType":"required",
+            "shape":"all"
+        },
+        "opFile":{
+            "value":"matmul_tik"
+        },
+        "opInterface":{
+            "value":"matmul_tik"
+        },
+        "output0":{
+            "dtype":"int32,int32,float",
+            "format":"ND,ND,ND",
+            "name":"y",
+            "paramType":"required",
+            "shape":"all"
+        }
+    }
+}

mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend910/aic-ascend910-ops-info.json

@@ -0,0 +1,158 @@
+{
+    "AddDSL":{
+        "input0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"x1",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"x2",
+            "paramType":"required"
+        },
+        "opFile":{
+            "value":"add_dsl"
+        },
+        "opInterface":{
+            "value":"add_dsl"
+        },
+        "output0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"y",
+            "paramType":"required"
+        }
+    },
+    "AddTik":{
+        "input0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"x1",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"x2",
+            "paramType":"required"
+        },
+        "opFile":{
+            "value":"add_tik"
+        },
+        "opInterface":{
+            "value":"add_tik"
+        },
+        "output0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"y",
+            "paramType":"required"
+        }
+    },
+    "FlashAttention":{
+        "input0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"q",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"k",
+            "paramType":"required"
+        },
+        "input2":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"v",
+            "paramType":"required"
+        },
+        "input3":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"attention_mask",
+            "paramType":"required"
+        },
+        "opFile":{
+            "value":"flash_attention"
+        },
+        "opInterface":{
+            "value":"flash_attention"
+        },
+        "output0":{
+            "dtype":"float16,float32,int32",
+            "format":"ND,ND,ND",
+            "name":"y",
+            "paramType":"required"
+        }
+    },
+    "KVCacheMgr":{
+        "input0":{
+            "dtype":"float32, int32, uint32, float16, int16, uint16, int8, uint8",
+            "format":"ND,ND,ND,ND,ND,ND,ND,ND",
+            "name":"past",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float32, int32, uint32, float16, int16, uint16, int8, uint8",
+            "format":"ND,ND,ND,ND,ND,ND,ND,ND",
+            "name":"cur",
+            "paramType":"required"
+        },
+        "input2":{
+            "dtype":"int32,int32,int32,int32,int32,int32,int32,int32",
+            "format":"ND,ND,ND,ND,ND,ND,ND,ND",
+            "name":"index",
+            "paramType":"required"
+        },
+        "needCheckSupport":{
+            "flag":"true"
+        },
+        "opFile":{
+            "value":"kv_cache_mgr"
+        },
+        "opInterface":{
+            "value":"kv_cache_mgr"
+        },
+        "output0":{
+            "dtype":"float32, int32, uint32, float16, int16, uint16, int8, uint8",
+            "format":"ND,ND,ND,ND,ND,ND,ND,ND",
+            "name":"past",
+            "paramType":"required"
+        }
+    },
+    "MatmulTik":{
+        "input0":{
+            "dtype":"int8,uint8,float16",
+            "format":"ND,ND,ND",
+            "name":"x1",
+            "needCompile":"false",
+            "paramType":"required",
+            "shape":"all"
+        },
+        "input1":{
+            "dtype":"int8,int8,float16",
+            "format":"ND,ND,ND",
+            "name":"x2",
+            "needCompile":"false",
+            "paramType":"required",
+            "shape":"all"
+        },
+        "opFile":{
+            "value":"matmul_tik"
+        },
+        "opInterface":{
+            "value":"matmul_tik"
+        },
+        "output0":{
+            "dtype":"int32,int32,float",
+            "format":"ND,ND,ND",
+            "name":"y",
+            "paramType":"required",
+            "shape":"all"
+        }
+    }
+}

mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend910b/aic-ascend910b-ops-info.json

@@ -0,0 +1,37 @@
+{
+    "KVCacheMgr":{
+        "input0":{
+            "dtype":"float32, int32, uint32, float16, int16, uint16, int8, uint8",
+            "format":"ND,ND,ND,ND,ND,ND,ND,ND",
+            "name":"past",
+            "paramType":"required"
+        },
+        "input1":{
+            "dtype":"float32, int32, uint32, float16, int16, uint16, int8, uint8",
+            "format":"ND,ND,ND,ND,ND,ND,ND,ND",
+            "name":"cur",
+            "paramType":"required"
+        },
+        "input2":{
+            "dtype":"int32,int32,int32,int32,int32,int32,int32,int32",
+            "format":"ND,ND,ND,ND,ND,ND,ND,ND",
+            "name":"index",
+            "paramType":"required"
+        },
+        "needCheckSupport":{
+            "flag":"true"
+        },
+        "opFile":{
+            "value":"kv_cache_mgr"
+        },
+        "opInterface":{
+            "value":"kv_cache_mgr"
+        },
+        "output0":{
+            "dtype":"float32, int32, uint32, float16, int16, uint16, int8, uint8",
+            "format":"ND,ND,ND,ND,ND,ND,ND,ND",
+            "name":"past",
+            "paramType":"required"
+        }
+    }
+}

mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/add_dsl.py

@@ -0,0 +1,46 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""ascend custom op: add by dsl"""
+import tbe.dsl as tbe
+from tbe import tvm
+from tbe.common.register import register_op_compute
+from tbe.common.utils import para_check
+
+
+@register_op_compute("add_dsl")
+def add_dsl_compute(x1, x2, y, kernel_name="add_dsl"):
+    res = tbe.vadd(x1, x2)
+    return res
+
+
+@para_check.check_op_params(para_check.REQUIRED_INPUT, para_check.REQUIRED_INPUT,
+                            para_check.REQUIRED_OUTPUT, para_check.KERNEL_NAME)
+def add_dsl(x1, x2, y, kernel_name="add_dsl"):
+    """add dsl impl function"""
+    data_x1 = tvm.placeholder(
+        x1.get("shape"), dtype=x1.get("dtype"), name="data_x1")
+    data_x2 = tvm.placeholder(
+        x2.get("shape"), dtype=x2.get("dtype"), name="data_x2")
+
+    res = add_dsl_compute(data_x1, data_x2, y, kernel_name)
+
+    # auto schedule
+    with tvm.target.cce():
+        schedule = tbe.auto_schedule(res)
+
+    # operator build
+    config = {"name": kernel_name,
+              "tensor_list": [data_x1, data_x2, res]}
+    tbe.build(schedule, config)

mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/add_tik.py

@@ -0,0 +1,51 @@
+# Copyright 2022 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""ascend custom op: add by tik"""
+from tbe.common.register import register_op_compute
+from tbe.common.utils import para_check
+from tbe import tik
+
+
+@register_op_compute("AddTik")
+@para_check.check_op_params(para_check.REQUIRED_INPUT, para_check.REQUIRED_INPUT,
+                            para_check.REQUIRED_OUTPUT, para_check.KERNEL_NAME)
+def add_tik(x1, x2, y, kernel_name="add_tik"):
+    """add dsl impl function"""
+    tik_instance = tik.Tik()
+    x1_shape = x1.get("shape")
+    x2_shape = x2.get("shape")
+    y_shape = y.get("shape")
+
+    data_a = tik_instance.Tensor(
+        "float16", x1_shape, name="x1", scope=tik.scope_gm)
+    data_b = tik_instance.Tensor(
+        "float16", x2_shape, name="x2", scope=tik.scope_gm)
+    data_c = tik_instance.Tensor(
+        "float16", y_shape, name="y", scope=tik.scope_gm)
+    data_a_ub = tik_instance.Tensor(
+        "float16", x1_shape, name="data_A_ub", scope=tik.scope_ubuf)
+    data_b_ub = tik_instance.Tensor(
+        "float16", x2_shape, name="data_B_ub", scope=tik.scope_ubuf)
+    data_c_ub = tik_instance.Tensor(
+        "float16", y_shape, name="data_C_ub", scope=tik.scope_ubuf)
+
+    tik_instance.data_move(data_a_ub, data_a, 0, 1, 128 // 16, 0, 0)
+    tik_instance.data_move(data_b_ub, data_b, 0, 1, 128 // 16, 0, 0)
+    tik_instance.vec_add(
+        128, data_c_ub[0], data_a_ub[0], data_b_ub[0], 1, 8, 8, 8)
+    tik_instance.data_move(data_c, data_c_ub, 0, 1, 128 // 16, 0, 0)
+    tik_instance.BuildCCE(kernel_name=kernel_name, inputs=[data_a, data_b], outputs=[data_c])
+
+    return tik_instance