mindspore 2.4.10__cp311-cp311-win_amd64.whl → 2.6.0__cp311-cp311-win_amd64.whl

mindspore/hal/stream.py

@@ -21,23 +21,28 @@ from mindspore._c_expression import default_stream as default_stream_
 from mindspore._c_expression import communication_stream as communication_stream_
 
 from mindspore import _checkparam as Validator
+from mindspore import log as logger
 from .event import Event
 
+function_stream_status = {'record_event': False, 'wait_event': False, 'wait_stream': False,
+                          'query': False, 'synchronize': False, 'set_cur_stream': False,
+                          'current_stream': False, 'default_stream': False,
+                          'communication_stream': False, 'StreamCtx': False}
 
 class Stream(Stream_):
     r"""
-    Wrapper around a device stream.
+    Wrapper around a device stream, this api will be deprecated and removed in future versions, please use
+    the api :class:`mindspore.runtime.Stream` instead.
 
     A device stream is a linear sequence of execution that belongs to a specific device,
     independent from other streams.
 
-    For a quick start of using Stream, please refer to `Illustration of stream management <https://www.mindspore.cn/docs/en/master/api_python/samples/hal/stream_manager.html>`_ .
-
     Args:
         priority (int, optional): priority of the stream, lower numbers represent higher priorities.
             By default, streams have priority ``0``.
         kwargs (dict): keyword arguments.
     """
+
     def __init__(self, priority=0, **kwargs):
         self.init_finished = False
         if 'stream' in kwargs and kwargs['stream'] is not None:
@@ -53,7 +58,7 @@ class Stream(Stream_):
 
     def record_event(self, event=None):
         r"""
-        Records an event.
+        Record an event.
 
         Args:
             event (Event, optional): event to record. If not given, a new one
@@ -62,21 +67,16 @@ class Stream(Stream_):
         Returns:
             Event, recorded event. If this argument is ``None``, a new one will be allocated. Default is ``None``.
 
-        Raises:
-            TypeError: If 'event' is neither a :class:`mindspore.hal.Event` nor a ``None``.
-
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> a = Tensor(np.ones([3, 3]), ms.float32)
-            >>> b = Tensor(np.ones([3, 3]), ms.float32)
-            >>> s1 = ms.hal.Stream()
-            >>> with ms.hal.StreamCtx(s1):
+            >>> import mindspore
+            >>> a = mindspore.tensor(mindspore.ops.ones([3, 3]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([3, 3]), mindspore.float32)
+            >>> s1 = mindspore.hal.Stream()
+            >>> with mindspore.hal.StreamCtx(s1):
             ...     c = a + b
             ...     event = s1.record_event()
             ...     d = a * b
-            >>> cur_stream = ms.hal.current_stream()
+            >>> cur_stream = mindspore.hal.current_stream()
             >>> cur_stream.wait_event(event)
             >>> e = c + 3
             >>> print(e)
@@ -84,6 +84,12 @@ class Stream(Stream_):
              [5. 5. 5.]
              [5. 5. 5.]]
         """
+        if not function_stream_status['record_event']:
+            function_stream_status['record_event'] = True
+            logger.warning(
+                "WARN_DEPRECATED: The usage of mindspore.hal.Stream().record_event() is deprecated."
+                " Please use mindspore.runtime.Stream().record_event()"
+            )
         if event is None:
             event = Event()
         if not isinstance(event, Event):
@@ -94,26 +100,21 @@ class Stream(Stream_):
 
     def wait_event(self, event):
         r"""
-        Makes all future work submitted to the stream wait for an event.
+        Make all future work submitted to the stream wait for an event.
 
         Args:
             event (Event): an event to wait for.
 
-        Raises:
-            TypeError: If 'event' is not a :class:`mindspore.hal.Event`.
-
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> a = Tensor(np.ones([3, 3]), ms.float32)
-            >>> b = Tensor(np.ones([3, 3]), ms.float32)
-            >>> s1 = ms.hal.Stream()
-            >>> with ms.hal.StreamCtx(s1):
+            >>> import mindspore
+            >>> a = mindspore.tensor(mindspore.ops.ones([3, 3]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([3, 3]), mindspore.float32)
+            >>> s1 = mindspore.hal.Stream()
+            >>> with mindspore.hal.StreamCtx(s1):
             ...     c = a + b
             ...     event = s1.record_event()
             ...     d = a * b
-            >>> cur_stream = ms.hal.current_stream()
+            >>> cur_stream = mindspore.hal.current_stream()
             >>> cur_stream.wait_event(event)
             >>> e = c + 3
             >>> print(e)
@@ -121,6 +122,12 @@ class Stream(Stream_):
              [5. 5. 5.]
              [5. 5. 5.]]
         """
+        if not function_stream_status['wait_event']:
+            function_stream_status['wait_event'] = True
+            logger.warning(
+                "WARN_DEPRECATED: The usage of mindspore.hal.current_stream().wait_event(event) is deprecated."
+                " Please use mindspore.runtime.current_stream().wait_event(event)"
+            )
         if not isinstance(event, Event):
             raise TypeError(f"For 'wait_event', the argument 'event' should be Event,"
                             f" but got {type(event)}.")
@@ -128,7 +135,7 @@ class Stream(Stream_):
 
     def wait_stream(self, stream):
         r"""
-        Synchronizes with another stream.
+        Synchronize with another stream.
 
         All future work submitted to this stream will wait until all kernels
         submitted to a given stream at the time of call complete.
@@ -136,26 +143,27 @@ class Stream(Stream_):
         Args:
             stream (Stream): a stream to synchronize.
 
-        Raises:
-            TypeError: If 'stream' is not a :class:`mindspore.hal.Stream`.
-
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> s1 = ms.hal.Stream()
-            >>> s2 = ms.hal.Stream()
-            >>> a = Tensor(np.ones([1, 2]), ms.float32)
-            >>> b = Tensor(np.ones([2, 2]), ms.float32)
-            >>> with ms.hal.StreamCtx(s1):
-            ...     c = ops.matmul(a, b)
-            >>> with ms.hal.StreamCtx(s2):
+            >>> import mindspore
+            >>> s1 = mindspore.hal.Stream()
+            >>> s2 = mindspore.hal.Stream()
+            >>> a = mindspore.tensor(mindspore.ops.ones([1, 2]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([2, 2]), mindspore.float32)
+            >>> with mindspore.hal.StreamCtx(s1):
+            ...     c = mindspore.ops.matmul(a, b)
+            >>> with mindspore.hal.StreamCtx(s2):
             ...     s2.wait_stream(s1)
-            ...     d = ops.matmul(c, b)
-            >>> ms.hal.synchronize()
+            ...     d = mindspore.ops.matmul(c, b)
+            >>> mindspore.hal.synchronize()
             >>> print(d)
             [[4. 4.]]
         """
+        if not function_stream_status['wait_stream']:
+            function_stream_status['wait_stream'] = True
+            logger.warning(
+                "WARN_DEPRECATED: The usage of mindspore.hal.Stream() is deprecated."
+                " Please use mindspore.runtime.Stream()"
+            )
         if not isinstance(stream, Stream):
             raise TypeError(f"For 'wait_stream', the argument 'stream' should be Stream,"
                             f" but got {type(stream)}.")
@@ -166,14 +174,12 @@ class Stream(Stream_):
         Wait for all the kernels in this stream to complete.
 
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> a = Tensor(np.ones([1024, 2048]), ms.float32)
-            >>> b = Tensor(np.ones([2048, 4096]), ms.float32)
-            >>> s1 = ms.hal.Stream()
-            >>> with ms.hal.StreamCtx(s1):
-            ...     c = ops.matmul(a, b)
+            >>> import mindspore
+            >>> a = mindspore.tensor(mindspore.ops.ones([1024, 2048]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([2048, 4096]), mindspore.float32)
+            >>> s1 = mindspore.hal.Stream()
+            >>> with mindspore.hal.StreamCtx(s1):
+            ...     c = mindspore.ops.matmul(a, b)
             >>> s1.synchronize()
             >>> assert s1.query()
         """
@@ -182,23 +188,27 @@ class Stream(Stream_):
 
     def query(self):
         r"""
-        Checks if all the work submitted has been completed.
+        Check if all the work submitted has been completed.
 
         Returns:
             A boolean indicating if all kernels in this stream are completed.
 
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> a = Tensor(np.ones([1024, 2048]), ms.float32)
-            >>> b = Tensor(np.ones([2048, 4096]), ms.float32)
-            >>> s1 = ms.hal.Stream()
-            >>> with ms.hal.StreamCtx(s1):
-            ...     c = ops.matmul(a, b)
+            >>> import mindspore
+            >>> a = mindspore.tensor(mindspore.ops.ones([1024, 2048]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([2048, 4096]), mindspore.float32)
+            >>> s1 = mindspore.hal.Stream()
+            >>> with mindspore.hal.StreamCtx(s1):
+            ...     c = mindspore.ops.matmul(a, b)
             >>> s1.synchronize()
             >>> assert s1.query()
         """
+        if not function_stream_status['query']:
+            function_stream_status['query'] = True
+            logger.warning(
+                "WARN_DEPRECATED: The usage of mindspore.hal.Stream() is deprecated."
+                " Please use mindspore.runtime.Stream()"
+            )
         # pylint: disable=useless-super-delegation
         return super().query()
 
@@ -219,44 +229,52 @@ class Stream(Stream_):
 
 def synchronize():
     r"""
-    Synchronize all streams on current device.(Each MindSpore process only occupies one device)
+    Synchronize all streams on current device, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.synchronize` instead.
 
     Examples:
-        >>> import mindspore as ms
-        >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> a = Tensor(np.ones([1024, 2048]), ms.float32)
-        >>> b = Tensor(np.ones([2048, 4096]), ms.float32)
-        >>> s1 = ms.hal.Stream()
-        >>> with ms.hal.StreamCtx(s1):
-        ...     c = ops.matmul(a, b)
-        >>> ms.hal.synchronize()
+        >>> import mindspore
+        >>> a = mindspore.tensor(mindspore.ops.ones([1024, 2048]), mindspore.float32)
+        >>> b = mindspore.tensor(mindspore.ops.ones([2048, 4096]), mindspore.float32)
+        >>> s1 = mindspore.hal.Stream()
+        >>> with mindspore.hal.StreamCtx(s1):
+        ...     c = mindspore.ops.matmul(a, b)
+        >>> mindspore.hal.synchronize()
         >>> assert s1.query()
     """
+    if not function_stream_status['synchronize']:
+        function_stream_status['synchronize'] = True
+        logger.warning(
+            "WARN_DEPRECATED: The usage of mindspore.hal.synchronize() is deprecated."
+            " Please use mindspore.runtime.synchronize()"
+        )
     synchronize_()
 
 
 def set_cur_stream(stream):
     r"""
-    Sets the current stream.This is a wrapper API to set the stream.
-    Usage of this function is discouraged in favor of the ``stream`` context manager.
+    Set the current stream, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.set_cur_stream` instead.
 
     Args:
         stream (Stream): selected stream. This function is a no-op
             if this argument is ``None``.
 
-    Raises:
-        TypeError: If 'stream' is neither a :class:`mindspore.hal.Stream` nor a ``None``.
-
     Examples:
-        >>> import mindspore as ms
-        >>> cur_stream = ms.hal.current_stream()
-        >>> assert cur_stream == ms.hal.default_stream()
-        >>> s1 = ms.hal.Stream()
-        >>> ms.hal.set_cur_stream(s1)
-        >>> assert ms.hal.current_stream() == s1
-        >>> ms.hal.set_cur_stream(ms.hal.default_stream())
+        >>> import mindspore
+        >>> cur_stream = mindspore.hal.current_stream()
+        >>> assert cur_stream == mindspore.hal.default_stream()
+        >>> s1 = mindspore.hal.Stream()
+        >>> mindspore.hal.set_cur_stream(s1)
+        >>> assert mindspore.hal.current_stream() == s1
+        >>> mindspore.hal.set_cur_stream(mindspore.hal.default_stream())
     """
+    if not function_stream_status['set_cur_stream']:
+        function_stream_status['set_cur_stream'] = True
+        logger.warning(
+            "WARN_DEPRECATED: The usage of mindspore.hal.Stream() is deprecated."
+            " Please use mindspore.runtime.Stream()"
+        )
     if stream is None:
         return
     if not isinstance(stream, Stream):
@@ -267,53 +285,74 @@ def set_cur_stream(stream):
 
 def current_stream():
     r"""
-    Return current stream used on this device.
+    Return current stream used on this device, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.current_stream` instead.
 
     Returns:
         stream (Stream), current stream.
 
     Examples:
-        >>> import mindspore as ms
-        >>> cur_stream = ms.hal.current_stream()
-        >>> assert cur_stream == ms.hal.default_stream()
+        >>> import mindspore
+        >>> cur_stream = mindspore.hal.current_stream()
+        >>> assert cur_stream == mindspore.hal.default_stream()
     """
+    if not function_stream_status['current_stream']:
+        function_stream_status['current_stream'] = True
+        logger.warning(
+            "WARN_DEPRECATED: The usage of mindspore.hal.current_stream() is deprecated."
+            " Please use mindspore.runtime.current_stream()"
+        )
     return Stream(stream=current_stream_())
 
 
 def default_stream():
     r"""
-    Return default stream on this device.
+    Return default stream on this device, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.default_stream` instead.
 
     Returns:
         stream (Stream), default stream.
 
     Examples:
-        >>> import mindspore as ms
-        >>> cur_stream = ms.hal.current_stream()
-        >>> assert cur_stream == ms.hal.default_stream()
+        >>> import mindspore
+        >>> cur_stream = mindspore.hal.current_stream()
+        >>> assert cur_stream == mindspore.hal.default_stream()
     """
+    if not function_stream_status['default_stream']:
+        function_stream_status['default_stream'] = True
+        logger.warning(
+            "WARN_DEPRECATED: The usage of mindspore.hal.default_stream() is deprecated."
+            " Please use mindspore.runtime.default_stream()"
+        )
     return Stream(stream=default_stream_())
 
 
 def communication_stream():
     r"""
-    Return default stream on this device.
+    Return communication stream on this device, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.communication_stream` instead.
 
     Returns:
-        stream (Stream), default stream.
+        stream (Stream), communication stream.
 
-    TODO
     Examples:
-        >>> import mindspore as ms
-        >>> cur_stream = ms.hal.current_stream()
-        >>> assert cur_stream == ms.hal.default_stream()
+        >>> import mindspore
+        >>> mindspore.hal.communication_stream()
+        Stream(device_name=Ascend, device_id:0, stream id:1)
     """
+    if not function_stream_status['communication_stream']:
+        function_stream_status['communication_stream'] = True
+        logger.warning(
+            "WARN_DEPRECATED: The usage of mindspore.hal.communication_stream() is deprecated."
+            " Please use mindspore.runtime.communication_stream()"
+        )
     return Stream(stream=communication_stream_())
 
 
 class StreamCtx():
     r"""
-    Context-manager that selects a given stream.
+    Context-manager that selects a given stream, this api will be deprecated and removed in future versions, please use
+    the api :class:`mindspore.runtime.StreamCtx` instead.
 
     All kernels queued within its context will be enqueued on a selected
     stream.
@@ -321,22 +360,24 @@ class StreamCtx():
     Args:
         ctx_stream (Stream): selected stream. This manager is a no-op if it's ``None``.
 
-    Raises:
-        TypeError: If 'stream' is neither a :class:`mindspore.hal.Stream` nor a ``None``.
-
     Examples:
-        >>> import mindspore as ms
-        >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> a = Tensor(np.ones([1024, 2048]), ms.float32)
-        >>> b = Tensor(np.ones([2048, 4096]), ms.float32)
-        >>> s1 = ms.hal.Stream()
-        >>> with ms.hal.StreamCtx(s1):
-        ...     c = ops.matmul(a, b)
-        >>> ms.hal.synchronize()
+        >>> import mindspore
+        >>> a = mindspore.tensor(mindspore.ops.ones([1024, 2048]), mindspore.float32)
+        >>> b = mindspore.tensor(mindspore.ops.ones([2048, 4096]), mindspore.float32)
+        >>> s1 = mindspore.hal.Stream()
+        >>> with mindspore.hal.StreamCtx(s1):
+        ...     c = mindspore.ops.matmul(a, b)
+        >>> mindspore.hal.synchronize()
         >>> assert s1.query()
     """
+
     def __init__(self, ctx_stream):
+        if not function_stream_status['StreamCtx']:
+            function_stream_status['StreamCtx'] = True
+            logger.warning(
+                "WARN_DEPRECATED: The usage of mindspore.hal.StreamCtx(s1) is deprecated."
+                " Please use mindspore.runtime.StreamCtx(s1)"
+            )
         if ctx_stream is not None and not isinstance(ctx_stream, Stream):
             raise TypeError(f"For 'StreamCtx', the argument 'ctx_stream' should be Stream,"
                             f" but got {type(ctx_stream)}.")

mindspore/include/api/context.h

@@ -27,7 +27,6 @@ namespace mindspore {
 enum DelegateMode {
   kNoDelegate = 0,
   kCoreML = 1,
-  kNNAPI = 2,
 };
 
 enum DeviceType {

mindspore/include/dataset/constants.h

@@ -129,10 +129,13 @@ enum class DATASET_API TensorImpl {
 
 /// \brief The mode for shuffling data.
 enum class DATASET_API ShuffleMode {
-  kFalse = 0,   ///< No shuffling is performed.
-  kFiles = 1,   ///< Shuffle files only.
-  kGlobal = 2,  ///< Shuffle both the files and samples.
-  kInfile = 3   ///< Shuffle data within each file.
+  kFalse = 0,    ///< No shuffling is performed.
+  kFiles = 1,    ///< Shuffle files only.
+  kGlobal = 2,   ///< Shuffle both the files and samples.
+  kInfile = 3,   ///< Shuffle data within each file.
+  kPartial = 4,  ///< Shuffle data with every 1 million samples
+  kAdaptive = 5  ///< When the number of dataset samples is less than or equal to 100 million, global shuffle is used.
+                 ///< When the number of dataset samples is greater than 100 million, partial shuffle is used.
 };
 
 /// \brief Possible scale for input audio.

mindspore/include/dataset/execute.h

@@ -26,7 +26,7 @@
 #include "include/api/visible.h"
 #include "include/dataset/constants.h"
 #include "include/dataset/transforms.h"
-#if !defined(BUILD_LITE) && defined(ENABLE_D)
+#if defined(ENABLE_D)
 #include "runtime/hardware/device_context.h"
 #include "runtime/hardware/device_context_manager.h"
 #endif
@@ -172,7 +172,7 @@ class DATASET_API Execute {
   struct ExtraInfo;
   std::shared_ptr<ExtraInfo> info_;
 
-#if !defined(BUILD_LITE) && defined(ENABLE_D)
+#if defined(ENABLE_D)
   // Ascend910B
   device::DeviceContext *device_context_ = nullptr;
   size_t stream_id_;

mindspore/log.py

@@ -26,6 +26,7 @@ from logging.handlers import RotatingFileHandler
 import traceback
 import threading
 import platform
+from datetime import datetime
 
 if platform.system() != "Windows":
     import fcntl
@@ -225,6 +226,55 @@ def critical(msg, *args, **kwargs):
     _get_logger().critical(msg, *args, **kwargs)
 
 
+def parse_vlog_level(s):
+    """Python parser for vlog level."""
+    if not s:
+        return False, ()
+    s = s.strip()
+    if not s.startswith('(') and not s.endswith(')'):
+        try:
+            level = int(s)
+            return True, (level, level)
+        except ValueError:
+            return False, ()
+    if s.startswith('(') and s.endswith(')'):
+        content = s[1:-1].strip()
+        parts = [p.strip() for p in content.split(',')]
+        if len(parts) != 2:
+            return False, ()
+        try:
+            if not parts[1]:
+                start = int(parts[0])
+                end = float('inf')
+            elif not parts[0]:
+                start = 1
+                end = int(parts[1])
+            else:
+                start = int(parts[0])
+                end = int(parts[1])
+            return True, (start, end)
+        except ValueError:
+            return False, ()
+    else:
+        return False, ()
+
+
+VLOG_VALID, VLOG_INDEX = parse_vlog_level(os.environ.get("VLOG_v"))
+
+
+def vlog_print(level, module, file, line, message):
+    """Read environment variable VLOG_v and print to log"""
+    global VLOG_VALID, VLOG_INDEX
+    target_level = int(level)
+    if VLOG_VALID and VLOG_INDEX[0] <= target_level <= VLOG_INDEX[1]:
+        now = datetime.now()
+        formatted_time = now.strftime("%Y-%m-%d-%H:%M:%S.%f")[:-3] + f".{now.microsecond // 1000}"
+        path = 'mindspore' + file.split("mindspore")[-1]
+        pid = os.getpid()
+        thread_id = threading.get_ident()
+        print(f"[V{level}] {module}({pid},{thread_id},python):{formatted_time} [{path}:{line}] {message}", flush=True)
+
+
 def get_level():
     """
     Get the logger level.

mindspore/mindrecord/__init__.py

@@ -15,13 +15,26 @@
 """
 Introduction of MindRecord.
 
-MindRecord is a module to implement reading, writing, searching and
-converting for MindSpore format dataset. Users could use the FileWriter
-API to generate MindRecord data and use the MindDataset API to load
-MindRecord data. Users could also convert other format datasets to
-mindrecord data through corresponding sub-module. You can also
-configure encryption, decryption, and integrity check to ensure
-the security of MindRecord datasets.
+MindRecord is an efficient data storage and reading module provided by MindSpore.
+This module provides several methods to help users convert various public datasets into the MindRecord format,
+as well as methods to read, write, and retrieve data from MindRecord files.
+
+.. image:: data_conversion_concept_en.png
+
+MindSpore format data allows for more convenient saving and loading of data,
+with the goal of normalizing user datasets and optimizing performance for different data scenarios.
+Using the MindRecord data format can reduce disk I/O and network I/O overhead,
+thereby providing a better data loading experience.
+
+Users can generate MindRecord format data files using `mindspore.mindrecord.FileWriter` and load MindRecord format
+datasets using `mindspore.dataset.MindDataset <https://www.mindspore.cn/docs/en/master/api_python/dataset/
+mindspore.dataset.MindDataset.html>`_ .
+
+Users can also convert datasets from other formats to the MindRecord format.
+For more details, please refer to `Converting Dataset to MindRecord <https://www.mindspore.cn/docs/en/master/
+model_train/dataset/record.html>`_ .
+Additionally, MindRecord supports file encryption, decryption,
+and integrity checks to ensure the security of MindRecord format datasets.
 """
 
 from .filewriter import FileWriter
@@ -39,5 +52,5 @@ from .config import *
 
 __all__ = ['FileWriter', 'FileReader', 'MindPage',
            'Cifar10ToMR', 'Cifar100ToMR', 'CsvToMR', 'ImageNetToMR', 'MnistToMR', 'TFRecordToMR',
-           'set_enc_key', 'set_enc_mode', 'set_dec_mode', 'set_hash_mode',
+           'set_enc_key', 'set_enc_mode', 'set_dec_mode',
            'SUCCESS', 'FAILED']