mindspore 2.6.0rc1__cp311-cp311-win_amd64.whl → 2.7.0__cp311-cp311-win_amd64.whl

mindspore/include/c_api/model_c.h

@@ -48,140 +48,111 @@ typedef bool (*MSKernelCallBackC)(const MSTensorHandleArray inputs, const MSTens
                                   const MSCallBackParamC kernel_Info);
 
 /// \brief Create a model object.
-///
 /// \return Model object handle.
 MS_API MSModelHandle MSModelCreate();
 
 /// \brief Destroy the model object.
-///
 /// \param[in] model Model object handle address.
 MS_API void MSModelDestroy(MSModelHandle *model);
 
 /// \brief Set workspace for the model object. Only valid for Iot.
-///
 /// \param[in] model Model object handle.
 /// \param[in] workspace Define the workspace address.
 /// \param[in] workspace_size Define the workspace size.
 MS_API void MSModelSetWorkspace(MSModelHandle model, void *workspace, size_t workspace_size);
 
 /// \brief Calculate the workspace size required for model inference. Only valid for Iot.
-///
 /// \param[in] model Model object handle.
 MS_API size_t MSModelCalcWorkspaceSize(MSModelHandle model);
 
 /// \brief Build the model from model file buffer so that it can run on a device.
-///
 /// \param[in] model Model object handle.
 /// \param[in] model_data Define the buffer read from a model file.
 /// \param[in] data_size Define bytes number of model file buffer.
 /// \param[in] model_type Define The type of model file.
 /// \param[in] model_context Define the context used to store options during execution.
-///
 /// \return MSStatus.
 MS_API MSStatus MSModelBuild(MSModelHandle model, const void *model_data, size_t data_size, MSModelType model_type,
                              const MSContextHandle model_context);
 
 /// \brief Load and build the model from model path so that it can run on a device.
-///
 /// \param[in] model Model object handle.
 /// \param[in] model_path Define the model file path.
 /// \param[in] model_type Define The type of model file.
 /// \param[in] model_context Define the context used to store options during execution.
-///
 /// \return MSStatus.
 MS_API MSStatus MSModelBuildFromFile(MSModelHandle model, const char *model_path, MSModelType model_type,
                                      const MSContextHandle model_context);
 
 /// \brief Resize the shapes of inputs.
-///
 /// \param[in] model Model object handle.
 /// \param[in] inputs The array that includes all input tensor handles.
 /// \param[in] shape_infos Defines the new shapes of inputs, should be consistent with inputs.
 /// \param[in] shape_info_num The num of shape_infos.
-///
 /// \return MSStatus.
 MS_API MSStatus MSModelResize(MSModelHandle model, const MSTensorHandleArray inputs, MSShapeInfo *shape_infos,
                               size_t shape_info_num);
 
 /// \brief Inference model.
-///
 /// \param[in] model Model object handle.
 /// \param[in] inputs The array that includes all input tensor handles.
 /// \param[out] outputs The array that includes all output tensor handles.
 /// \param[in] before CallBack before predict.
 /// \param[in] after CallBack after predict.
-///
 /// \return MSStatus.
 MS_API MSStatus MSModelPredict(MSModelHandle model, const MSTensorHandleArray inputs, MSTensorHandleArray *outputs,
                                const MSKernelCallBackC before, const MSKernelCallBackC after);
 
 /// \brief Run model by step. Only valid for Iot.
-///
 /// \param[in] model Model object handle.
 /// \param[in] before CallBack before RunStep.
 /// \param[in] after CallBack after RunStep.
-///
 /// \return MSStatus.
 MS_API MSStatus MSModelRunStep(MSModelHandle model, const MSKernelCallBackC before, const MSKernelCallBackC after);
 
 /// \brief Set the model running mode. Only valid for Iot.
-///
 /// \param[in] model Model object handle.
 /// \param[in] train True means model runs in Train Mode, otherwise Eval Mode.
-///
 /// \return Status of operation.
 MS_API MSStatus MSModelSetTrainMode(const MSModelHandle model, bool train);
 
 /// \brief Export the weights of model to the binary file. Only valid for Iot.
-///
 /// \param[in] model Model object handle.
 /// \param[in] export_path Define the export weight file path.
-///
 /// \return Status of operation.
 MS_API MSStatus MSModelExportWeight(const MSModelHandle model, const char *export_path);
 
 /// \brief Obtain all input tensor handles of the model.
-///
 /// \param[in] model Model object handle.
-///
 /// \return The array that includes all input tensor handles.
 MS_API MSTensorHandleArray MSModelGetInputs(const MSModelHandle model);
 
 /// \brief Obtain all output tensor handles of the model.
-///
 /// \param[in] model Model object handle.
-///
 /// \return The array that includes all output tensor handles.
 MS_API MSTensorHandleArray MSModelGetOutputs(const MSModelHandle model);
 
 /// \brief Obtain the input tensor handle of the model by name.
-///
 /// \param[in] model Model object handle.
 /// \param[in] tensor_name The name of tensor.
-///
 /// \return The input tensor handle with the given name, if the name is not found, an NULL is returned.
 MS_API MSTensorHandle MSModelGetInputByTensorName(const MSModelHandle model, const char *tensor_name);
 
 /// \brief Obtain the output tensor handle of the model by name.
-///
 /// \param[in] model Model object handle.
 /// \param[in] tensor_name The name of tensor.
-///
 /// \return The output tensor handle with the given name, if the name is not found, an NULL is returned.
 MS_API MSTensorHandle MSModelGetOutputByTensorName(const MSModelHandle model, const char *tensor_name);
 
 /// \brief Create a TrainCfg object. Only valid for Lite Train.
-///
 /// \return TrainCfg object handle.
 MS_API MSTrainCfgHandle MSTrainCfgCreate();
 
 /// \brief Destroy the train_cfg object. Only valid for Lite Train.
-///
 /// \param[in] train_cfg TrainCfg object handle.
 MS_API void MSTrainCfgDestroy(MSTrainCfgHandle *train_cfg);
 
 /// \brief Obtains part of the name that identify a loss kernel. Only valid for Lite Train.
-///
 /// \param[in] train_cfg TrainCfg object handle.
 /// \param[in] num The num of loss_name.
 ///
@@ -189,112 +160,88 @@ MS_API void MSTrainCfgDestroy(MSTrainCfgHandle *train_cfg);
 MS_API char **MSTrainCfgGetLossName(MSTrainCfgHandle train_cfg, size_t *num);
 
 /// \brief Set part of the name that identify a loss kernel. Only valid for Lite Train.
-///
 /// \param[in] train_cfg TrainCfg object handle.
 /// \param[in] loss_name define part of the name that identify a loss kernel.
 /// \param[in] num The num of loss_name.
 MS_API void MSTrainCfgSetLossName(MSTrainCfgHandle train_cfg, const char **loss_name, size_t num);
 
 /// \brief Obtains optimization level of the train_cfg. Only valid for Lite Train.
-///
 /// \param[in] train_cfg TrainCfg object handle.
-///
 /// \return MSOptimizationLevel.
 MS_API MSOptimizationLevel MSTrainCfgGetOptimizationLevel(MSTrainCfgHandle train_cfg);
 
 /// \brief Set optimization level of the train_cfg. Only valid for Lite Train.
-///
 /// \param[in] train_cfg TrainCfg object handle.
 /// \param[in] level The optimization level of train_cfg.
 MS_API void MSTrainCfgSetOptimizationLevel(MSTrainCfgHandle train_cfg, MSOptimizationLevel level);
 
 /// \brief Build the train model from model buffer so that it can run on a device. Only valid for Lite Train.
-///
 /// \param[in] model Model object handle.
 /// \param[in] model_data Define the buffer read from a model file.
 /// \param[in] data_size Define bytes number of model file buffer.
 /// \param[in] model_type Define The type of model file.
 /// \param[in] model_context Define the context used to store options during execution.
 /// \param[in] train_cfg Define the config used by training.
-///
 /// \return MSStatus.
 MS_API MSStatus MSTrainModelBuild(MSModelHandle model, const void *model_data, size_t data_size, MSModelType model_type,
                                   const MSContextHandle model_context, const MSTrainCfgHandle train_cfg);
 
 /// \brief Build the train model from model file buffer so that it can run on a device. Only valid for Lite Train.
-///
 /// \param[in] model Model object handle.
 /// \param[in] model_path Define the model path.
 /// \param[in] model_type Define The type of model file.
 /// \param[in] model_context Define the context used to store options during execution.
 /// \param[in] train_cfg Define the config used by training.
-///
 /// \return MSStatus.
 MS_API MSStatus MSTrainModelBuildFromFile(MSModelHandle model, const char *model_path, MSModelType model_type,
                                           const MSContextHandle model_context, const MSTrainCfgHandle train_cfg);
 
 /// \brief Train model by step. Only valid for Lite Train.
-///
 /// \param[in] model Model object handle.
 /// \param[in] before CallBack before predict.
 /// \param[in] after CallBack after predict.
-///
 /// \return MSStatus.
 MS_API MSStatus MSRunStep(MSModelHandle model, const MSKernelCallBackC before, const MSKernelCallBackC after);
 
 /// \brief Sets the Learning Rate of the training. Only valid for Lite Train.
-///
 /// \param[in] learning_rate to set.
-///
 /// \return MSStatus of operation.
 MS_API MSStatus MSModelSetLearningRate(MSModelHandle model, float learning_rate);
 
 /// \brief Obtains the Learning Rate of the optimizer. Only valid for Lite Train.
-///
 /// \return Learning rate. 0.0 if no optimizer was found.
 MS_API float MSModelGetLearningRate(MSModelHandle model);
 
 /// \brief Obtains all weights tensors of the model. Only valid for Lite Train.
-///
 /// \param[in] model Model object handle.
-///
 /// \return The vector that includes all gradient tensors.
 MS_API MSTensorHandleArray MSModelGetWeights(MSModelHandle model);
 
 /// \brief update weights tensors of the model. Only valid for Lite Train.
-///
 /// \param[in] new_weights A vector new weights.
-///
 /// \return MSStatus
 MS_API MSStatus MSModelUpdateWeights(MSModelHandle model, const MSTensorHandleArray new_weights);
 
 /// \brief Get the model running mode.
-///
 /// \param[in] model Model object handle.
-///
 /// \return Is Train Mode or not.
 MS_API bool MSModelGetTrainMode(MSModelHandle model);
 
 /// \brief Set the model running mode. Only valid for Lite Train.
-///
 /// \param[in] model Model object handle.
 /// \param[in] train True means model runs in Train Mode, otherwise Eval Mode.
-///
 /// \return MSStatus.
 MS_API MSStatus MSModelSetTrainMode(MSModelHandle model, bool train);
 
 /// \brief Setup training with virtual batches. Only valid for Lite Train.
-///
 /// \param[in] model Model object handle.
 /// \param[in] virtual_batch_multiplier - virtual batch multiplier, use any number < 1 to disable.
 /// \param[in] lr - learning rate to use for virtual batch, -1 for internal configuration.
 /// \param[in] momentum - batch norm momentum to use for virtual batch, -1 for internal configuration.
-///
 /// \return MSStatus.
 MS_API MSStatus MSModelSetupVirtualBatch(MSModelHandle model, int virtual_batch_multiplier, float lr, float momentum);
 
 /// \brief Export training model from file. Only valid for Lite Train.
-///
 /// \param[in] model The model data.
 /// \param[in] model_type The model file type.
 /// \param[in] model_file The exported model file.
@@ -303,14 +250,12 @@ MS_API MSStatus MSModelSetupVirtualBatch(MSModelHandle model, int virtual_batch_
 /// \param[in] output_tensor_name The set the name of the output tensor of the exported reasoning model, default as
 /// empty, and export the complete reasoning model.
 /// \param[in] num The number of output_tensor_name.
-///
 /// \return MSStatus.
 MS_API MSStatus MSExportModel(MSModelHandle model, MSModelType model_type, const char *model_file,
                               MSQuantizationType quantization_type, bool export_inference_only,
                               char **output_tensor_name, size_t num);
 
 /// \brief Export training model from buffer. Only valid for Lite Train.
-///
 /// \param[in] model The model data.
 /// \param[in] model_type The model file type.
 /// \param[in] model_data The exported model buffer.
@@ -320,14 +265,12 @@ MS_API MSStatus MSExportModel(MSModelHandle model, MSModelType model_type, const
 /// \param[in] output_tensor_name The set the name of the output tensor of the exported reasoning model, default as
 /// empty, and export the complete reasoning model.
 /// \param[in] num The number of output_tensor_name.
-///
 /// \return MSStatus.
 MS_API MSStatus MSExportModelBuffer(MSModelHandle model, MSModelType model_type, char **model_data, size_t *data_size,
                                     MSQuantizationType quantization_type, bool export_inference_only,
                                     char **output_tensor_name, size_t num);
 
 /// \brief Export model's weights, which can be used in micro only. Only valid for Lite Train.
-///
 /// \param[in] model The model data.
 /// \param[in] model_type The model file type.
 /// \param[in] weight_file The path of exported weight file.
@@ -335,7 +278,6 @@ MS_API MSStatus MSExportModelBuffer(MSModelHandle model, MSModelType model_type,
 /// \param[in] enable_fp16 Float-weight is whether to be saved in float16 format.
 /// \param[in] changeable_weights_name The set the name of these weight tensors, whose shape is changeable.
 /// \param[in] num The number of changeable_weights_name.
-///
 /// \return MSStatus.
 MS_API MSStatus MSExportWeightsCollaborateWithMicro(MSModelHandle model, MSModelType model_type,
                                                     const char *weight_file, bool is_inference, bool enable_fp16,

mindspore/include/c_api/tensor_c.h

@@ -28,115 +28,89 @@ extern "C" {
 typedef void *MSTensorHandle;
 
 /// \brief Create a tensor object.
-///
 /// \param[in] name The name of the tensor.
 /// \param[in] type The data type of the tensor.
 /// \param[in] shape The shape of the tensor.
 /// \param[in] shape_num The num of the shape.
 /// \param[in] data The data pointer that points to allocated memory.
 /// \param[in] data_len The length of the memory, in bytes.
-///
 /// \return Tensor object handle.
 MS_API MSTensorHandle MSTensorCreate(const char *name, MSDataType type, const int64_t *shape, size_t shape_num,
                                      const void *data, size_t data_len);
 
 /// \brief Destroy the tensor object.
-///
 /// \param[in] tensor Tensor object handle address.
 MS_API void MSTensorDestroy(MSTensorHandle *tensor);
 
 /// \brief Obtain a deep copy of the tensor.
-///
 /// \param[in] tensor Tensor object handle.
-///
 /// \return Tensor object handle.
 MS_API MSTensorHandle MSTensorClone(MSTensorHandle tensor);
 
 /// \brief Set the name for the tensor.
-///
 /// \param[in] tensor Tensor object handle.
 /// \param[in] name The name of the tensor.
 MS_API void MSTensorSetName(MSTensorHandle tensor, const char *name);
 
 /// \brief Obtain the name of the tensor.
-///
 /// \param[in] tensor Tensor object handle.
-///
 /// \return The name of the tensor.
 MS_API const char *MSTensorGetName(const MSTensorHandle tensor);
 
 /// \brief Set the data type for the tensor.
-///
 /// \param[in] tensor Tensor object handle.
 /// \param[in] type The data type of the tensor.
 MS_API void MSTensorSetDataType(MSTensorHandle tensor, MSDataType type);
 
 /// \brief Obtain the data type of the tensor.
-///
 /// \param[in] tensor Tensor object handle.
-///
 /// \return The date type of the tensor.
 MS_API MSDataType MSTensorGetDataType(const MSTensorHandle tensor);
 
 /// \brief Set the shape for the tensor.
-///
 /// \param[in] tensor Tensor object handle.
 /// \param[in] shape The shape array.
 /// \param[in] shape_num Dimension of shape.
 MS_API void MSTensorSetShape(MSTensorHandle tensor, const int64_t *shape, size_t shape_num);
 
 /// \brief Obtain the shape of the tensor.
-///
 /// \param[in] tensor Tensor object handle.
 /// \param[out] shape_num Dimension of shape.
-///
 /// \return The shape array of the tensor.
 MS_API const int64_t *MSTensorGetShape(const MSTensorHandle tensor, size_t *shape_num);
 
 /// \brief Set the format for the tensor.
-///
 /// \param[in] tensor Tensor object handle.
 /// \param[in] format The format of the tensor.
 MS_API void MSTensorSetFormat(MSTensorHandle tensor, MSFormat format);
 
 /// \brief Obtain the format of the tensor.
-///
 /// \param[in] tensor Tensor object handle.
-///
 /// \return The format of the tensor.
 MS_API MSFormat MSTensorGetFormat(const MSTensorHandle tensor);
 
 /// \brief Obtain the data for the tensor.
-///
 /// \param[in] tensor Tensor object handle.
 /// \param[in] data A pointer to the data of the tensor.
 MS_API void MSTensorSetData(MSTensorHandle tensor, void *data);
 
 /// \brief Obtain the data pointer of the tensor.
-///
 /// \param[in] tensor Tensor object handle.
-///
 /// \return The data pointer of the tensor.
 MS_API const void *MSTensorGetData(const MSTensorHandle tensor);
 
 /// \brief Obtain the mutable data pointer of the tensor. If the internal data is empty, it will allocate memory.
-///
 /// \param[in] tensor Tensor object handle.
-///
 /// \return The data pointer of the tensor.
 MS_API void *MSTensorGetMutableData(const MSTensorHandle tensor);
 
 /// \brief Obtain the element number of the tensor.
-///
 /// \param[in] tensor Tensor object handle.
-///
 /// \return The element number of the tensor.
 MS_API int64_t MSTensorGetElementNum(const MSTensorHandle tensor);
 
 /// \brief Obtain the data size fo the tensor.
-///
 /// \param[in] tensor Tensor object handle.
-///
 /// \return The data size of the tensor.
 MS_API size_t MSTensorGetDataSize(const MSTensorHandle tensor);
 

mindspore/include/dataset/constants.h

@@ -356,6 +356,15 @@ constexpr int32_t kMaxLegalPort = 65535;
 // Invalid OpenCV type should not be from 0 to 7 (opencv4/opencv2/core/hal/interface.h)
 constexpr uint8_t kCVInvalidType = 255;
 
+constexpr size_t dimension_zero = 0;
+constexpr size_t dimension_one = 1;
+constexpr size_t dimension_two = 2;
+constexpr size_t dimension_three = 3;
+constexpr size_t size_one = 1;
+constexpr size_t size_two = 2;
+constexpr size_t size_three = 3;
+constexpr size_t size_four = 4;
+
 using connection_id_type = uint64_t;
 using session_id_type = uint32_t;
 using row_id_type = int64_t;

mindspore/include/dataset/vision_ascend.h

@@ -31,7 +31,7 @@ namespace mindspore {
 namespace dataset {
 // Transform operations for performing computer vision.
 namespace vision {
-/* ##################################### API class ###########################################*/
+/* ##################################### API class ########################################### */
 
 /// \brief Decode and resize JPEG image using the hardware algorithm of
 ///     Ascend series chip DVPP module.

mindspore/mindrecord/tools/cifar10.py

@@ -15,24 +15,73 @@
 """
 Cifar10 reader class.
 """
-import builtins
+import hashlib
 import io
+import os
 import pickle
 import re
-import os
 import numpy as np
 
+from mindspore import log as logger
 from ..shardutils import check_filename
 
 __all__ = ['Cifar10']
 
-safe_builtins = {
-    'range',
-    'complex',
-    'set',
-    'frozenset',
-    'slice',
-}
+
+class CifarMD5Validator:
+    '''
+    MD5 check for cifar10-batch-py files
+    '''
+    def __init__(self):
+        self.md5_map = {'data_batch_1': 'c99cafc152244af753f735de768cd75f',
+                        'data_batch_2': 'd4bba439e000b95fd0a9bffe97cbabec',
+                        'data_batch_3': '54ebc095f3ab1f0389bbae665268c751',
+                        'data_batch_4': '634d18415352ddfa80567beed471001a',
+                        'data_batch_5': '482c414d41f54cd18b22e5b47cb7c3cb',
+                        'test_batch': '40351d587109b95175f43aff81a1287e'}
+
+    @staticmethod
+    def calculate_md5(file_path):
+        """
+        Calculate MD5 hash of a file.
+
+        Args:
+            file_path (str): Path to the file to calculate MD5 for.
+
+        Returns:
+            MD5 hash string if successful, None otherwise.
+        """
+        try:
+            with open(file_path, 'rb') as f:
+                file_hash = hashlib.md5()
+                chunk = f.read(8192)
+                while chunk:
+                    file_hash.update(chunk)
+                    chunk = f.read(8192)
+            return file_hash.hexdigest()
+        except (IOError, OSError):
+            return None
+
+    def check(self, file_path, file_name):
+        """
+        Check if the file's MD5 matches the expected value.
+
+        Args:
+            file_path (str): Path to the file to check.
+            file_name (str): Key in md5_map to identify the expected MD5.
+
+        Returns:
+            True if MD5 matches, False otherwise (including if file doesn't exist).
+
+        Raises:
+            KeyError: If file_name is not found in md5_map.
+        """
+        expected_md5 = self.md5_map.get(file_name)
+        actual_md5 = CifarMD5Validator.calculate_md5(os.path.join(file_path, file_name))
+
+        if actual_md5 is None or expected_md5 is None or actual_md5 != expected_md5:
+            logger.warning(f"The MD5 value of {file_name} does not match the official CIFAR10 file."
+                           "This file may pose a security risk.")
 
 
 class RestrictedUnpickler(pickle.Unpickler):
@@ -44,8 +93,6 @@ class RestrictedUnpickler(pickle.Unpickler):
     """
     def find_class(self, module, name):
         # Only allow safe classes from builtins and numpy
-        if module == "builtins" and name in safe_builtins:
-            return getattr(builtins, name)
         if module == "numpy.core.multiarray" and name == "_reconstruct":
             return getattr(np.core.multiarray, name)
         if module == "numpy":
@@ -89,6 +136,7 @@ class Cifar10:
         self.one_hot = one_hot
         self.images = None
         self.labels = None
+        self.validator = CifarMD5Validator()
 
     def load_data(self):
         """
@@ -104,12 +152,14 @@ class Cifar10:
         for file in files:
             if re.match("data_batch_*", file):
                 real_file_path = os.path.realpath(self.path)
+                self.validator.check(real_file_path, file)
                 with open(os.path.join(real_file_path, file), 'rb') as f: # load train data
                     dic = restricted_loads(f.read())
                     images = np.r_[images, dic[b"data"].reshape([-1, 3, 32, 32])]
                     labels.append(dic[b"labels"])
             elif re.match("test_batch", file):                       # load test data
                 real_file_path = os.path.realpath(self.path)
+                self.validator.check(real_file_path, file)
                 with open(os.path.join(real_file_path, file), 'rb') as f:
                     dic = restricted_loads(f.read())
                     test_images = np.array(dic[b"data"].reshape([-1, 3, 32, 32]))

mindspore/mindrecord/tools/cifar10_to_mr.py

@@ -107,6 +107,11 @@ class Cifar10ToMR:
         Note:
             Please refer to the Examples of :class:`mindspore.mindrecord.Cifar10ToMR` .
 
+        .. warning::
+            `Cifar10ToMR.transform()` uses `pickle` module implicitly, which is known to be insecure.
+            It is possible to construct malicious pickle data which will execute arbitrary code during unpickling.
+            Never load data that could have come from an untrusted source, or that could have been tampered with.
+
         Args:
             fields (list[str], optional): A list of index fields. Default: ``None`` . For index field settings,
                 please refer to :func:`mindspore.mindrecord.FileWriter.add_index` .