mindspore 2.3.0__cp39-cp39-win_amd64.whl → 2.4.0__cp39-cp39-win_amd64.whl

mindspore/hal/contiguous_tensors_handle.py

@@ -0,0 +1,175 @@
+# This is the Python alloc contiguous memory handle.
+#
+# Copyright 2024-2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Contiguous memory handle."""
+from mindspore.common.tensor import Tensor
+from mindspore.common.api import _convert_python_data
+from mindspore.common.dtype import type_size_in_bytes
+from mindspore._c_expression import slice_by_tensor_index, slice_by_padding_shape, \
+    combine_tensor_list_contiguous as combine_tensor_list, Tensor as Tensor_
+
+
+def combine_tensor_list_contiguous(tensor_list, enable_mem_align=True):
+    r"""
+    Return a contiguous memory handle where contiguous memory has been requested and slicing functionality is provided.
+
+    Args:
+        tensor_list (list[Tensor], Tuple[Tensor]): The tensor list to be stored.
+        enable_mem_align (bool, optional): Whether to enable the memory alignment function.
+            False is not supported. Default: ``True`` .
+
+    Returns:
+            ContiguousTensorsHandle, a manager with contiguous memory.
+
+    Examples:
+        >>> import numpy as np
+        >>> import mindspore as ms
+        >>> from mindspore import Tensor
+        >>> from mindspore.hal.contiguous_tensors_handle import combine_tensor_list_contiguous
+        >>> x = Tensor(np.array([1, 2, 3]).astype(np.float32))
+        >>> y = Tensor(np.array([4, 5, 6]).astype(np.float32))
+        >>> handle = combine_tensor_list_contiguous([x, y], True)
+        >>> print(handle[0].shape)
+        [1]
+        >>> print(handle[1: 3].asnumpy())
+        [2, 3]
+        >>> print(output.slice_by_tensor_index(0, 1).asnumpy())
+        [1, 2, 3]
+    """
+    return ContiguousTensorsHandle(tensor_list, enable_mem_align)
+
+
+class ContiguousTensorsHandle:
+    r"""
+    ContiguousTensorsHandle is a handle manage continuous memory.
+
+    Args:
+        tensor_list (list[Tensor], Tuple[Tensor]): The tensor list to be stored.
+        enable_mem_align (bool, optional): Whether to enable the memory alignment function.
+            False is not supported. Default: ``True`` .
+
+    Returns:
+        ContiguousTensorsHandle, a manager with contiguous memory.
+
+    Examples:
+        >>> import numpy as np
+        >>> import mindspore as ms
+        >>> from mindspore import Tensor
+        >>> from mindspore.hal.contiguous_tensors_handle import ContiguousTensorsHandle
+        >>> x = Tensor(np.array([1, 2, 3]).astype(np.float32))
+        >>> y = Tensor(np.array([4, 5, 6]).astype(np.float32))
+        >>> handle = ContiguousTensorsHandle([x, y], True)
+        >>> print(handle[0].shape)
+        [1]
+        >>> print(handle[1: 3].asnumpy())
+        [2, 3]
+    """
+
+    def __init__(self, tensor_list, enable_mem_align=True):
+        if isinstance(tensor_list, (list, tuple)):
+            for tensor in tensor_list:
+                if not isinstance(tensor, (Tensor, Tensor_)):
+                    raise TypeError(f"input list must be [Tensor, ...].")
+            if isinstance(tensor_list, list):
+                self.tensor_list = tuple(tensor_list)
+            else:
+                self.tensor_list = tensor_list
+        else:
+            raise TypeError(f"input list must be [Tensor, ...].")
+        if not isinstance(enable_mem_align, bool):
+            raise TypeError(f"enable_mem_align must be bool.")
+        padding_sizes_pair = combine_tensor_list(self.tensor_list, enable_mem_align)
+        self.before_padding_sizes = padding_sizes_pair[0]
+        self.after_padding_sizes = padding_sizes_pair[1]
+        self.total_padding_size = sum(self.after_padding_sizes)
+        self.handle_shape = self.total_padding_size / type_size_in_bytes(self.tensor_list[0].dtype)
+        self.enable_mem_align = enable_mem_align
+
+    def __getitem__(self, item):
+        """
+        item is sliced by shape
+        :param item:
+        :return: Tensor
+        """
+        start = 0
+        end = int(self.handle_shape)
+        if isinstance(item, slice):
+            if item.start is not None:
+                start = item.start
+            if item.stop is not None:
+                end = item.stop
+            if not isinstance(start, int) or not isinstance(end, int):
+                raise TypeError(f"slice input error.")
+            if start < 0 or end > self.handle_shape or start >= end:
+                raise ValueError(f"slice input error.")
+            return _convert_python_data(slice_by_padding_shape(self.tensor_list[0], start, end))
+        if not isinstance(item, int):
+            raise TypeError(f"slice input must be "
+                            f"1.index -> int."
+                            f"2.[start: end: step] -> [int: int: int].")
+        if item < 0 or item > self.handle_shape:
+            raise ValueError(f"slice input is out of tensor_list size.")
+        return _convert_python_data(slice_by_padding_shape(self.tensor_list[0], item, item + 1))
+
+    def __str__(self):
+        list_str = "Handle total size: " + str(self.total_padding_size) + "\n"
+        index = 0
+        for tensor in self.tensor_list:
+            list_str = list_str + "Tensor[" + str(index) + "]: " + str(tensor.asnumpy()) + "\n"
+            index += 1
+        return list_str
+
+    def slice_by_tensor_index(self, start=None, end=None):
+        """
+        Return the tensor which is sliced by tensor index.
+
+        Args:
+            start(int, None): Starting position. Default：``None``.
+            end(int, None): Deadline position. Default：``None``.
+
+        Returns:
+            Tensor，is sliced by tensor index.
+
+        Raises:
+            TypeError: If `start` or `end`, is neither an 'int' nor a 'none'.
+            ValueError: If values of `start` or `end` are negative, or out of the list range, or start >= end.
+
+        Examples:
+            >>> import numpy as np
+            >>> import mindspore as ms
+            >>> from mindspore import Tensor
+            >>> from mindspore.hal.contiguous_tensors_handle import ContiguousTensorsHandle
+            >>> x = Tensor(np.array([1, 2, 3]).astype(np.float32))
+            >>> y = Tensor(np.array([4, 5, 6]).astype(np.float32))
+            >>> handle = ContiguousTensorsHandle([x, y], True)
+            >>> print(output.slice_by_tensor_index(0, 1).asnumpy())
+            [1, 2, 3]
+        """
+        index_start = 0
+        index_end = len(self.tensor_list)
+        if start is not None:
+            index_start = start
+            if end is None:
+                index_end = index_start + 1
+        if end is not None:
+            index_end = end
+        if not isinstance(index_start, int) or not isinstance(index_end, int):
+            raise TypeError(f"slice input error.")
+
+        if index_start < 0 or index_end > len(self.tensor_list) or index_start >= index_end:
+            raise ValueError(f"slice input error.")
+        return _convert_python_data(slice_by_tensor_index(self.tensor_list, self.before_padding_sizes,
+                                                          self.after_padding_sizes, index_start, index_end))

mindspore/hal/stream.py

@@ -18,6 +18,8 @@ from mindspore._c_expression import set_cur_stream as set_cur_stream_
 from mindspore._c_expression import synchronize as synchronize_
 from mindspore._c_expression import current_stream as current_stream_
 from mindspore._c_expression import default_stream as default_stream_
+from mindspore._c_expression import communication_stream as communication_stream_
+
 from mindspore import _checkparam as Validator
 from .event import Event
 
@@ -293,6 +295,22 @@ def default_stream():
     return Stream(stream=default_stream_())
 
 
+def communication_stream():
+    r"""
+    Return default stream on this device.
+
+    Returns:
+        stream (Stream), default stream.
+
+    TODO
+    Examples:
+        >>> import mindspore as ms
+        >>> cur_stream = ms.hal.current_stream()
+        >>> assert cur_stream == ms.hal.default_stream()
+    """
+    return Stream(stream=communication_stream_())
+
+
 class StreamCtx():
     r"""
     Context-manager that selects a given stream.

mindspore/include/api/model_group.h

@@ -33,8 +33,10 @@ class ModelGroupImpl;
 /// multiple models to share workspace memory.
 
 enum class ModelGroupFlag : int {
+  kUnknown = 0x0000,
   kShareWeight = 0x0001,
   kShareWorkspace = 0x0002,
+  kShareWeightAndWorkspace = 0x0003,
 };
 
 class MS_API ModelGroup {
@@ -47,7 +49,7 @@ class MS_API ModelGroup {
   /// \param[in] model_path_list Define the list of model path.
   ///
   /// \return Status.
-  Status AddModel(const std::vector<std::string> &model_path_list);
+  inline Status AddModel(const std::vector<std::string> &model_path_list);
 
   /// \brief Add models that require shared workspace memory.
   ///
@@ -74,6 +76,16 @@ class MS_API ModelGroup {
 
  private:
   std::shared_ptr<ModelGroupImpl> impl_;
+  Status AddModel(const std::vector<std::vector<char>> &model_path_list);
 };
+
+Status ModelGroup::AddModel(const std::vector<std::string> &model_path_list) {
+  std::vector<std::vector<char>> model_path_list_str;
+  for (auto str : model_path_list) {
+    model_path_list_str.push_back(StringToChar(str));
+  }
+  return AddModel(model_path_list_str);
+}
+
 }  // namespace mindspore
 #endif  // MINDSPORE_INCLUDE_API_MODEL_GROUP_H

mindspore/include/api/types.h

@@ -60,6 +60,16 @@ struct QuantParam {
   double max;          ///< Quantization max value
 };
 
+using Key = struct MS_API Key {
+  size_t max_key_len = 32;
+  size_t len = 0;
+  unsigned char key[32] = {0};
+  Key() : len(0) {}
+  explicit Key(const char *dec_key, size_t key_len);
+};
+
+constexpr char kDecModeAesGcm[] = "AES-GCM";
+
 class Allocator;
 /// \brief The MSTensor class defines a tensor in MindSpore.
 class MS_API MSTensor {
@@ -402,16 +412,6 @@ std::string MSTensor::Name() const { return CharToString(CharName()); }
 
 void MSTensor::SetTensorName(const std::string &name) { SetTensorName(StringToChar(name)); }
 
-using Key = struct MS_API Key {
-  const size_t max_key_len = 32;
-  size_t len = 0;
-  unsigned char key[32] = {0};
-  Key() : len(0) {}
-  explicit Key(const char *dec_key, size_t key_len);
-};
-
-constexpr char kDecModeAesGcm[] = "AES-GCM";
-
 /// \brief CallBackParam defined input arguments for callBack function.
 struct MSCallBackParam {
   std::string node_name; /**< node name argument */

mindspore/include/dataset/config.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020-2022 Huawei Technologies Co., Ltd
+ * Copyright 2020-2024 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,7 @@
 #include <vector>
 
 #include "include/api/dual_abi_helper.h"
-#include "include/api/types.h"
+#include "include/api/visible.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/include/dataset/constants.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020-2022 Huawei Technologies Co., Ltd
+ * Copyright 2020-2024 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@
 #include <limits>
 #include <random>
 
-#include "include/api/types.h"
+#include "include/api/visible.h"
 
 namespace mindspore {
 namespace dataset {

mindspore/include/dataset/execute.h

@@ -1,5 +1,5 @@
 /**
- * Copyright 2020-2023 Huawei Technologies Co., Ltd
+ * Copyright 2020-2024 Huawei Technologies Co., Ltd
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,7 @@
 #include <vector>
 
 #include "include/api/context.h"
-#include "include/api/types.h"
+#include "include/api/visible.h"
 #include "include/dataset/constants.h"
 #include "include/dataset/transforms.h"
 #if !defined(BUILD_LITE) && defined(ENABLE_D)

mindspore/include/dataset/vision.h

@@ -441,6 +441,7 @@ class DATASET_API CutOut final : public TensorTransform {
   std::shared_ptr<Data> data_;
 };
 
+#ifdef ENABLE_FFMPEG
 /// \brief Decode the input video.
 class DATASET_API DecodeVideo final : public TensorTransform {
  public:
@@ -470,6 +471,7 @@ class DATASET_API DecodeVideo final : public TensorTransform {
   /// \return Shared pointer to TensorOperation object.
   std::shared_ptr<TensorOperation> Parse() override;
 };
+#endif
 
 /// \brief Encode the image as JPEG data.
 /// \param[in] image The image to be encoded.
@@ -1717,6 +1719,7 @@ Status DATASET_API ReadFile(const std::string &filename, mindspore::MSTensor *ou
 Status DATASET_API ReadImage(const std::string &filename, mindspore::MSTensor *output,
                              ImageReadMode mode = ImageReadMode::kUNCHANGED);
 
+#ifdef ENABLE_FFMPEG
 /// \brief Read the video, audio, metadata from a video file. It supports AVI, H264, H265, MOV, MP4, WMV file formats.
 /// \param[in] filename The path to the videoe file to be read.
 /// \param[out] video_output The video frames of the video file.
@@ -1737,6 +1740,7 @@ Status DATASET_API ReadVideo(const std::string &filename, mindspore::MSTensor *v
 /// \return The status code.
 Status DATASET_API ReadVideoTimestamps(const std::string &filename, std::tuple<std::vector<float>, float> *output,
                                        const std::string &pts_unit = "pts");
+#endif
 
 /// \brief Crop the given image and zoom to the specified size.
 class DATASET_API ResizedCrop final : public TensorTransform {

mindspore/log.py

@@ -507,7 +507,7 @@ def _create_logfile_dir(kwargs):
     rank_id = _get_rank_id()
     log_dir += '/rank_' + rank_id + '/logs'
     if not os.path.exists(log_dir):
-        os.makedirs(log_dir, exist_ok=True)
+        os.makedirs(log_dir, mode=0o700, exist_ok=True)
     return log_dir
 
 

mindspore/mindrecord/filewriter.py

@@ -113,8 +113,10 @@ class FileWriter:
         # parallel write mode
         self._parallel_writer = None
         self._writers = None
-        self._queue = None
+        self._queues = []      # the data queue from main process to worker process
+        self._msg_queues = []  # the err msg from worker process to main process
         self._workers = None
+        self._workers_loop_index = 0
         self._index_workers = None
 
     @classmethod
@@ -340,42 +342,53 @@ class FileWriter:
         # init the _writers and launch the workers
         if self._writers is None:
             self._writers = [None] * len(self._paths)  # writers used by worker
-            self._queue = mp.Queue(len(self._paths) * 2)  # queue for worker
+            self._queues = [mp.Queue(2) for _ in range(len(self._paths))]  # data queue for worker
+            self._msg_queues = [mp.Queue(2) for _ in range(len(self._paths))]  # msg queue for worker
             self._workers = [None] * len(self._paths)  # worker process
+            worker_pid_list = []
             for i, path in enumerate(self._paths):
                 self._writers[i] = ShardWriter()
                 self._writers[i].open([path], self._overwrite)
                 self._writers[i].set_shard_header(self._header)
 
                 # launch the workers for parallel write
-                self._queue._joincancelled = True  # pylint: disable=W0212
-                p = mp.Process(target=self._write_worker, name="WriterWorker" + str(i), args=(i, self._queue))
+                self._queues[i]._joincancelled = True  # pylint: disable=W0212
+                self._msg_queues[i]._joincancelled = True  # pylint: disable=W0212
+                p = mp.Process(target=self._write_worker, name="WriterWorker" + str(i),
+                               args=(i, self._queues[i], self._msg_queues[i]))
                 p.daemon = True
                 p.start()
                 logger.info("Start worker process(pid:{}) to parallel write.".format(p.pid))
                 self._workers[i] = p
+                worker_pid_list.append(self._workers[i].pid)
 
-        # fill the self._queue
+        # fill the self._queues
         check_interval = 0.5  # 0.5s
         start_time = time.time()
         while True:
             try:
-                self._queue.put(raw_data, block=False)
+                self._queues[self._workers_loop_index].put(raw_data, block=False)
             except queue.Full:
                 if time.time() - start_time > check_interval:
                     start_time = time.time()
                     logger.warning("Because there are too few MindRecord file shards, the efficiency of parallel " \
                                    "writing is too low. You can stop the current task and add the parameter " \
                                    "`shard_num` of `FileWriter` to upgrade the task.")
-
-                # check the status of worker process
-                for i in range(len(self._paths)):
-                    if not self._workers[i].is_alive():
-                        raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
-                                           "the above log".format(self._workers[i].pid))
+                self.check_worker_status(self._workers_loop_index)
                 continue
+            self._workers_loop_index = (self._workers_loop_index + 1) % len(self._paths)
             return
 
+    def check_all_worker_status(self):
+        for index in range(len(self._msg_queues)):
+            self.check_worker_status(index)
+
+    def check_worker_status(self, index):
+        if not self._msg_queues[index].empty():
+            if self._msg_queues[index].get() == "Error":
+                raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
+                                   "the above log".format(self._workers[index].pid))
+
     def set_header_size(self, header_size):
         """
         Set the size of header which contains shard information, schema information, \
@@ -495,52 +508,45 @@ class FileWriter:
 
     def _parallel_commit(self):
         """Parallel commit"""
-        # if some workers stopped, error may occur
-        alive_count = 0
-        for i in range(len(self._paths)):
-            if self._workers[i].is_alive():
-                alive_count += 1
-        if alive_count != len(self._paths):
-            raise RuntimeError("Parallel write worker error, please check the above log.")
+        # waiting for all the self._queues had been writed by workers
+        for index in range(len(self._paths)):
+            while not self._queues[index].empty():
+                logger.info("Waiting for the data is writed by worker: {}.".format(self._workers[index].pid))
+                time.sleep(1)
+
+                # check the workers status
+                self.check_worker_status(index)
 
         # send EOF to worker process
-        for i in range(len(self._paths)):
+        for index in range(len(self._paths)):
+            logger.info("Send EOF flag to worker: {}.".format(self._workers[index].pid))
             while True:
                 try:
-                    self._queue.put("EOF", block=False)
+                    self._queues[index].put("EOF", block=False)
                 except queue.Full:
+                    # check the workers status
+                    self.check_worker_status(index)
+
                     time.sleep(1)
-                    if not self._workers[i].is_alive():
-                        raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
-                                           "the above log".format(self._workers[i].pid))
                     continue
                 break
 
-        # wait the worker processing
-        while True:
-            alive_count = 0
-            for i in range(len(self._paths)):
-                if self._workers[i].is_alive():
-                    alive_count += 1
-            if alive_count == 0:
-                break
-            time.sleep(1)
-            logger.info("Waiting for all the parallel workers to finish.")
-
-        del self._queue
-
-        # wait for worker process stop
-        for index in range(len(self._paths)):
+        worker_sucess = 0
+        for index in range(len(self._msg_queues)):
             while True:
-                logger.info("Waiting for the worker process(pid:{}) to process all the data.".format(
-                    self._workers[index].pid))
-                if self._workers[index].is_alive():
-                    time.sleep(1)
-                    continue
-                elif self._workers[index].exitcode != 0:
-                    raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
-                                       "the above log".format(self._workers[index].pid))
-                break
+                logger.info("Waiting for the worker: {} to exit.".format(self._workers[index].pid))
+                if not self._msg_queues[index].empty():
+                    ret = self._msg_queues[index].get()
+                    if ret == "Success":
+                        worker_sucess += 1
+                        break
+                    else:
+                        raise RuntimeError("Worker process(pid:{}) has stopped abnormally. Please check " \
+                                           "the above log".format(self._workers[index].pid))
+                time.sleep(1)
+
+        del self._queues
+        del self._msg_queues
 
         if self._index_generator:
             # use parallel index workers to generator index
@@ -678,7 +684,7 @@ class FileWriter:
                 return False, error
         return True, error
 
-    def _write_worker(self, i, in_queue):
+    def _write_worker(self, i, in_queue, msg_queue):
         """The worker do the data check and write to disk for parallel mode"""
         while True:
             # try to get new raw_data from master
@@ -691,15 +697,26 @@ class FileWriter:
             if raw_data == "EOF":
                 ret = self._writers[i].commit()
                 if ret != SUCCESS:
+                    msg_queue.put("Error")
                     raise RuntimeError("Commit the {}th shard of MindRecord file failed.".format(i))
+
+                logger.info("Send Success flag from worker: {} to master: {}.".format(os.getpid(), os.getppid()))
+                msg_queue.put("Success")
+
                 break
 
             # check the raw_data
             if not isinstance(raw_data, list):
+                msg_queue.put("Error")
                 raise ParamTypeError('raw_data', 'list')
             for each_raw in raw_data:
                 if not isinstance(each_raw, dict):
+                    msg_queue.put("Error")
                     raise ParamTypeError('raw_data item', 'dict')
 
-            self._verify_based_on_schema(raw_data)
-            self._writers[i].write_raw_data(raw_data, True, False)
+            try:
+                self._verify_based_on_schema(raw_data)
+                self._writers[i].write_raw_data(raw_data, True, False)
+            except Exception as e:
+                msg_queue.put("Error")
+                raise e