mindspore 2.5.0__cp311-cp311-win_amd64.whl → 2.6.0rc1__cp311-cp311-win_amd64.whl

mindspore/hal/stream.py

@@ -31,14 +31,12 @@ function_stream_status = {'record_event': False, 'wait_event': False, 'wait_stre
 
 class Stream(Stream_):
     r"""
-    Wrapper around a device stream.
+    Wrapper around a device stream, this api will be deprecated and removed in future versions, please use
+    the api :class:`mindspore.runtime.Stream` instead.
 
     A device stream is a linear sequence of execution that belongs to a specific device,
     independent from other streams.
 
-    Note:
-        - The api will be deprecated, please use the api :class:`mindspore.runtime.Stream`.
-
     Args:
         priority (int, optional): priority of the stream, lower numbers represent higher priorities.
             By default, streams have priority ``0``.
@@ -60,7 +58,7 @@ class Stream(Stream_):
 
     def record_event(self, event=None):
         r"""
-        Records an event.
+        Record an event.
 
         Args:
             event (Event, optional): event to record. If not given, a new one
@@ -69,21 +67,16 @@ class Stream(Stream_):
         Returns:
             Event, recorded event. If this argument is ``None``, a new one will be allocated. Default is ``None``.
 
-        Raises:
-            TypeError: If 'event' is neither a :class:`mindspore.hal.Event` nor a ``None``.
-
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> a = Tensor(np.ones([3, 3]), ms.float32)
-            >>> b = Tensor(np.ones([3, 3]), ms.float32)
-            >>> s1 = ms.hal.Stream()
-            >>> with ms.hal.StreamCtx(s1):
+            >>> import mindspore
+            >>> a = mindspore.tensor(mindspore.ops.ones([3, 3]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([3, 3]), mindspore.float32)
+            >>> s1 = mindspore.hal.Stream()
+            >>> with mindspore.hal.StreamCtx(s1):
             ...     c = a + b
             ...     event = s1.record_event()
             ...     d = a * b
-            >>> cur_stream = ms.hal.current_stream()
+            >>> cur_stream = mindspore.hal.current_stream()
             >>> cur_stream.wait_event(event)
             >>> e = c + 3
             >>> print(e)
@@ -107,26 +100,21 @@ class Stream(Stream_):
 
     def wait_event(self, event):
         r"""
-        Makes all future work submitted to the stream wait for an event.
+        Make all future work submitted to the stream wait for an event.
 
         Args:
             event (Event): an event to wait for.
 
-        Raises:
-            TypeError: If 'event' is not a :class:`mindspore.hal.Event`.
-
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> a = Tensor(np.ones([3, 3]), ms.float32)
-            >>> b = Tensor(np.ones([3, 3]), ms.float32)
-            >>> s1 = ms.hal.Stream()
-            >>> with ms.hal.StreamCtx(s1):
+            >>> import mindspore
+            >>> a = mindspore.tensor(mindspore.ops.ones([3, 3]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([3, 3]), mindspore.float32)
+            >>> s1 = mindspore.hal.Stream()
+            >>> with mindspore.hal.StreamCtx(s1):
             ...     c = a + b
             ...     event = s1.record_event()
             ...     d = a * b
-            >>> cur_stream = ms.hal.current_stream()
+            >>> cur_stream = mindspore.hal.current_stream()
             >>> cur_stream.wait_event(event)
             >>> e = c + 3
             >>> print(e)
@@ -147,7 +135,7 @@ class Stream(Stream_):
 
     def wait_stream(self, stream):
         r"""
-        Synchronizes with another stream.
+        Synchronize with another stream.
 
         All future work submitted to this stream will wait until all kernels
         submitted to a given stream at the time of call complete.
@@ -155,23 +143,18 @@ class Stream(Stream_):
         Args:
             stream (Stream): a stream to synchronize.
 
-        Raises:
-            TypeError: If 'stream' is not a :class:`mindspore.hal.Stream`.
-
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> s1 = ms.hal.Stream()
-            >>> s2 = ms.hal.Stream()
-            >>> a = Tensor(np.ones([1, 2]), ms.float32)
-            >>> b = Tensor(np.ones([2, 2]), ms.float32)
-            >>> with ms.hal.StreamCtx(s1):
-            ...     c = ops.matmul(a, b)
-            >>> with ms.hal.StreamCtx(s2):
+            >>> import mindspore
+            >>> s1 = mindspore.hal.Stream()
+            >>> s2 = mindspore.hal.Stream()
+            >>> a = mindspore.tensor(mindspore.ops.ones([1, 2]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([2, 2]), mindspore.float32)
+            >>> with mindspore.hal.StreamCtx(s1):
+            ...     c = mindspore.ops.matmul(a, b)
+            >>> with mindspore.hal.StreamCtx(s2):
             ...     s2.wait_stream(s1)
-            ...     d = ops.matmul(c, b)
-            >>> ms.hal.synchronize()
+            ...     d = mindspore.ops.matmul(c, b)
+            >>> mindspore.hal.synchronize()
             >>> print(d)
             [[4. 4.]]
         """
@@ -191,14 +174,12 @@ class Stream(Stream_):
         Wait for all the kernels in this stream to complete.
 
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> a = Tensor(np.ones([1024, 2048]), ms.float32)
-            >>> b = Tensor(np.ones([2048, 4096]), ms.float32)
-            >>> s1 = ms.hal.Stream()
-            >>> with ms.hal.StreamCtx(s1):
-            ...     c = ops.matmul(a, b)
+            >>> import mindspore
+            >>> a = mindspore.tensor(mindspore.ops.ones([1024, 2048]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([2048, 4096]), mindspore.float32)
+            >>> s1 = mindspore.hal.Stream()
+            >>> with mindspore.hal.StreamCtx(s1):
+            ...     c = mindspore.ops.matmul(a, b)
             >>> s1.synchronize()
             >>> assert s1.query()
         """
@@ -207,20 +188,18 @@ class Stream(Stream_):
 
     def query(self):
         r"""
-        Checks if all the work submitted has been completed.
+        Check if all the work submitted has been completed.
 
         Returns:
             A boolean indicating if all kernels in this stream are completed.
 
         Examples:
-            >>> import mindspore as ms
-            >>> import numpy as np
-            >>> from mindspore import Tensor, ops
-            >>> a = Tensor(np.ones([1024, 2048]), ms.float32)
-            >>> b = Tensor(np.ones([2048, 4096]), ms.float32)
-            >>> s1 = ms.hal.Stream()
-            >>> with ms.hal.StreamCtx(s1):
-            ...     c = ops.matmul(a, b)
+            >>> import mindspore
+            >>> a = mindspore.tensor(mindspore.ops.ones([1024, 2048]), mindspore.float32)
+            >>> b = mindspore.tensor(mindspore.ops.ones([2048, 4096]), mindspore.float32)
+            >>> s1 = mindspore.hal.Stream()
+            >>> with mindspore.hal.StreamCtx(s1):
+            ...     c = mindspore.ops.matmul(a, b)
             >>> s1.synchronize()
             >>> assert s1.query()
         """
@@ -250,21 +229,17 @@ class Stream(Stream_):
 
 def synchronize():
     r"""
-    Synchronize all streams on current device.(Each MindSpore process only occupies one device)
-
-    Note:
-        - The api will be deprecated, please use the api :func:`mindspore.runtime.synchronize` instead.
+    Synchronize all streams on current device, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.synchronize` instead.
 
     Examples:
-        >>> import mindspore as ms
-        >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> a = Tensor(np.ones([1024, 2048]), ms.float32)
-        >>> b = Tensor(np.ones([2048, 4096]), ms.float32)
-        >>> s1 = ms.hal.Stream()
-        >>> with ms.hal.StreamCtx(s1):
-        ...     c = ops.matmul(a, b)
-        >>> ms.hal.synchronize()
+        >>> import mindspore
+        >>> a = mindspore.tensor(mindspore.ops.ones([1024, 2048]), mindspore.float32)
+        >>> b = mindspore.tensor(mindspore.ops.ones([2048, 4096]), mindspore.float32)
+        >>> s1 = mindspore.hal.Stream()
+        >>> with mindspore.hal.StreamCtx(s1):
+        ...     c = mindspore.ops.matmul(a, b)
+        >>> mindspore.hal.synchronize()
         >>> assert s1.query()
     """
     if not function_stream_status['synchronize']:
@@ -278,27 +253,21 @@ def synchronize():
 
 def set_cur_stream(stream):
     r"""
-    Sets the current stream.This is a wrapper API to set the stream.
-    Usage of this function is discouraged in favor of the ``stream`` context manager.
-
-    Note:
-        - The api will be deprecated, please use the api :func:`mindspore.runtime.set_cur_stream` instead.
+    Set the current stream, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.set_cur_stream` instead.
 
     Args:
         stream (Stream): selected stream. This function is a no-op
             if this argument is ``None``.
 
-    Raises:
-        TypeError: If 'stream' is neither a :class:`mindspore.hal.Stream` nor a ``None``.
-
     Examples:
-        >>> import mindspore as ms
-        >>> cur_stream = ms.hal.current_stream()
-        >>> assert cur_stream == ms.hal.default_stream()
-        >>> s1 = ms.hal.Stream()
-        >>> ms.hal.set_cur_stream(s1)
-        >>> assert ms.hal.current_stream() == s1
-        >>> ms.hal.set_cur_stream(ms.hal.default_stream())
+        >>> import mindspore
+        >>> cur_stream = mindspore.hal.current_stream()
+        >>> assert cur_stream == mindspore.hal.default_stream()
+        >>> s1 = mindspore.hal.Stream()
+        >>> mindspore.hal.set_cur_stream(s1)
+        >>> assert mindspore.hal.current_stream() == s1
+        >>> mindspore.hal.set_cur_stream(mindspore.hal.default_stream())
     """
     if not function_stream_status['set_cur_stream']:
         function_stream_status['set_cur_stream'] = True
@@ -316,18 +285,16 @@ def set_cur_stream(stream):
 
 def current_stream():
     r"""
-    Return current stream used on this device.
-
-    Note:
-        - The api will be deprecated, please use the api :func:`mindspore.runtime.current_stream` instead.
+    Return current stream used on this device, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.current_stream` instead.
 
     Returns:
         stream (Stream), current stream.
 
     Examples:
-        >>> import mindspore as ms
-        >>> cur_stream = ms.hal.current_stream()
-        >>> assert cur_stream == ms.hal.default_stream()
+        >>> import mindspore
+        >>> cur_stream = mindspore.hal.current_stream()
+        >>> assert cur_stream == mindspore.hal.default_stream()
     """
     if not function_stream_status['current_stream']:
         function_stream_status['current_stream'] = True
@@ -340,18 +307,16 @@ def current_stream():
 
 def default_stream():
     r"""
-    Return default stream on this device.
-
-    Note:
-        - The api will be deprecated, please use the api :func:`mindspore.runtime.default_stream` instead.
+    Return default stream on this device, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.default_stream` instead.
 
     Returns:
         stream (Stream), default stream.
 
     Examples:
-        >>> import mindspore as ms
-        >>> cur_stream = ms.hal.current_stream()
-        >>> assert cur_stream == ms.hal.default_stream()
+        >>> import mindspore
+        >>> cur_stream = mindspore.hal.current_stream()
+        >>> assert cur_stream == mindspore.hal.default_stream()
     """
     if not function_stream_status['default_stream']:
         function_stream_status['default_stream'] = True
@@ -364,17 +329,15 @@ def default_stream():
 
 def communication_stream():
     r"""
-    Return communication stream on this device.
-
-    Note:
-        - The api will be deprecated, please use the api :func:`mindspore.runtime.communication_stream` instead.
+    Return communication stream on this device, this api will be deprecated and removed in future versions, please use
+    the api :func:`mindspore.runtime.communication_stream` instead.
 
     Returns:
         stream (Stream), communication stream.
 
     Examples:
-        >>> import mindspore as ms
-        >>> ms.hal.communication_stream()
+        >>> import mindspore
+        >>> mindspore.hal.communication_stream()
         Stream(device_name=Ascend, device_id:0, stream id:1)
     """
     if not function_stream_status['communication_stream']:
@@ -388,10 +351,8 @@ def communication_stream():
 
 class StreamCtx():
     r"""
-    Context-manager that selects a given stream.
-
-    Note:
-        - The api will be deprecated, please use the api :class:`mindspore.runtime.StreamCtx`.
+    Context-manager that selects a given stream, this api will be deprecated and removed in future versions, please use
+    the api :class:`mindspore.runtime.StreamCtx` instead.
 
     All kernels queued within its context will be enqueued on a selected
     stream.
@@ -399,19 +360,14 @@ class StreamCtx():
     Args:
         ctx_stream (Stream): selected stream. This manager is a no-op if it's ``None``.
 
-    Raises:
-        TypeError: If 'stream' is neither a :class:`mindspore.hal.Stream` nor a ``None``.
-
     Examples:
-        >>> import mindspore as ms
-        >>> import numpy as np
-        >>> from mindspore import Tensor, ops
-        >>> a = Tensor(np.ones([1024, 2048]), ms.float32)
-        >>> b = Tensor(np.ones([2048, 4096]), ms.float32)
-        >>> s1 = ms.hal.Stream()
-        >>> with ms.hal.StreamCtx(s1):
-        ...     c = ops.matmul(a, b)
-        >>> ms.hal.synchronize()
+        >>> import mindspore
+        >>> a = mindspore.tensor(mindspore.ops.ones([1024, 2048]), mindspore.float32)
+        >>> b = mindspore.tensor(mindspore.ops.ones([2048, 4096]), mindspore.float32)
+        >>> s1 = mindspore.hal.Stream()
+        >>> with mindspore.hal.StreamCtx(s1):
+        ...     c = mindspore.ops.matmul(a, b)
+        >>> mindspore.hal.synchronize()
         >>> assert s1.query()
     """
 

mindspore/include/dataset/constants.h

@@ -129,10 +129,13 @@ enum class DATASET_API TensorImpl {
 
 /// \brief The mode for shuffling data.
 enum class DATASET_API ShuffleMode {
-  kFalse = 0,   ///< No shuffling is performed.
-  kFiles = 1,   ///< Shuffle files only.
-  kGlobal = 2,  ///< Shuffle both the files and samples.
-  kInfile = 3   ///< Shuffle data within each file.
+  kFalse = 0,    ///< No shuffling is performed.
+  kFiles = 1,    ///< Shuffle files only.
+  kGlobal = 2,   ///< Shuffle both the files and samples.
+  kInfile = 3,   ///< Shuffle data within each file.
+  kPartial = 4,  ///< Shuffle data with every 1 million samples
+  kAdaptive = 5  ///< When the number of dataset samples is less than or equal to 100 million, global shuffle is used.
+                 ///< When the number of dataset samples is greater than 100 million, partial shuffle is used.
 };
 
 /// \brief Possible scale for input audio.

mindspore/include/dataset/execute.h

@@ -26,7 +26,7 @@
 #include "include/api/visible.h"
 #include "include/dataset/constants.h"
 #include "include/dataset/transforms.h"
-#if !defined(BUILD_LITE) && defined(ENABLE_D)
+#if defined(ENABLE_D)
 #include "runtime/hardware/device_context.h"
 #include "runtime/hardware/device_context_manager.h"
 #endif
@@ -172,7 +172,7 @@ class DATASET_API Execute {
   struct ExtraInfo;
   std::shared_ptr<ExtraInfo> info_;
 
-#if !defined(BUILD_LITE) && defined(ENABLE_D)
+#if defined(ENABLE_D)
   // Ascend910B
   device::DeviceContext *device_context_ = nullptr;
   size_t stream_id_;

mindspore/log.py

@@ -226,9 +226,47 @@ def critical(msg, *args, **kwargs):
     _get_logger().critical(msg, *args, **kwargs)
 
 
+def parse_vlog_level(s):
+    """Python parser for vlog level."""
+    if not s:
+        return False, ()
+    s = s.strip()
+    if not s.startswith('(') and not s.endswith(')'):
+        try:
+            level = int(s)
+            return True, (level, level)
+        except ValueError:
+            return False, ()
+    if s.startswith('(') and s.endswith(')'):
+        content = s[1:-1].strip()
+        parts = [p.strip() for p in content.split(',')]
+        if len(parts) != 2:
+            return False, ()
+        try:
+            if not parts[1]:
+                start = int(parts[0])
+                end = float('inf')
+            elif not parts[0]:
+                start = 1
+                end = int(parts[1])
+            else:
+                start = int(parts[0])
+                end = int(parts[1])
+            return True, (start, end)
+        except ValueError:
+            return False, ()
+    else:
+        return False, ()
+
+
+VLOG_VALID, VLOG_INDEX = parse_vlog_level(os.environ.get("VLOG_v"))
+
+
 def vlog_print(level, module, file, line, message):
-    '''Read environment variable VLOG_v and print to log'''
-    if os.environ.get("VLOG_v") == level:
+    """Read environment variable VLOG_v and print to log"""
+    global VLOG_VALID, VLOG_INDEX
+    target_level = int(level)
+    if VLOG_VALID and VLOG_INDEX[0] <= target_level <= VLOG_INDEX[1]:
         now = datetime.now()
         formatted_time = now.strftime("%Y-%m-%d-%H:%M:%S.%f")[:-3] + f".{now.microsecond // 1000}"
         path = 'mindspore' + file.split("mindspore")[-1]

mindspore/mindrecord/__init__.py

@@ -15,13 +15,26 @@
 """
 Introduction of MindRecord.
 
-MindRecord is a module to implement reading, writing, searching and
-converting for MindSpore format dataset. Users could use the FileWriter
-API to generate MindRecord data and use the MindDataset API to load
-MindRecord data. Users could also convert other format datasets to
-mindrecord data through corresponding sub-module. You can also
-configure encryption, decryption, and integrity check to ensure
-the security of MindRecord datasets.
+MindRecord is an efficient data storage and reading module provided by MindSpore.
+This module provides several methods to help users convert various public datasets into the MindRecord format,
+as well as methods to read, write, and retrieve data from MindRecord files.
+
+.. image:: data_conversion_concept_en.png
+
+MindSpore format data allows for more convenient saving and loading of data,
+with the goal of normalizing user datasets and optimizing performance for different data scenarios.
+Using the MindRecord data format can reduce disk I/O and network I/O overhead,
+thereby providing a better data loading experience.
+
+Users can generate MindRecord format data files using `mindspore.mindrecord.FileWriter` and load MindRecord format
+datasets using `mindspore.dataset.MindDataset <https://www.mindspore.cn/docs/en/master/api_python/dataset/
+mindspore.dataset.MindDataset.html>`_ .
+
+Users can also convert datasets from other formats to the MindRecord format.
+For more details, please refer to `Converting Dataset to MindRecord <https://www.mindspore.cn/docs/en/master/
+model_train/dataset/record.html>`_ .
+Additionally, MindRecord supports file encryption, decryption,
+and integrity checks to ensure the security of MindRecord format datasets.
 """
 
 from .filewriter import FileWriter