mindspore 2.1.0__cp38-none-any.whl → 2.2.10__cp38-none-any.whl

mindspore/dataset/transforms/__init__.py

@@ -31,7 +31,7 @@ Note: Legacy c_transforms and py_transforms are deprecated but can still be impo
     from mindspore.dataset.transforms import py_transforms
 
 See `Common Transforms
-<https://www.mindspore.cn/tutorials/en/r2.1/beginner/transforms.html#common-transforms>`_ tutorial for more details.
+<https://www.mindspore.cn/tutorials/en/r2.2/beginner/transforms.html#common-transforms>`_ tutorial for more details.
 
 Descriptions of common data processing terms are as follows:
 

mindspore/dataset/transforms/py_transforms_util.py

@@ -18,11 +18,12 @@ Built-in py_transforms_utils functions.
 import json
 import random
 from enum import IntEnum
-from types import FunctionType
+from types import FunctionType, MethodType
 import numpy as np
 
 from mindspore import log as logger
 from ..core.py_util_helpers import is_numpy, ExceptionHandler
+from .. import transforms as t
 
 
 class Implementation(IntEnum):
@@ -212,16 +213,43 @@ class FuncWrapper:
 
     def to_json(self):
         """ Serialize to JSON format """
-        if isinstance(self.transform, FunctionType):
-            # User-defined Python functions cannot be fully nor correctly serialized.
-            # Log a warning, and produce minimal info for the Python UDF, so that serialization of the
-            # dataset pipeline can continue.
-            # Note that the serialized JSON output is not valid to be deserialized.
-            logger.warning("Serialization of user-defined Python functions is not supported. "
-                           "Any produced serialized JSON file output for this dataset pipeline is not valid "
-                           "to be deserialized.")
+        # User-defined Python functions cannot be fully nor correctly serialized.
+        # Log a warning, and produce minimal info for the Python UDF, so that serialization of the
+        # dataset pipeline can continue.
+        # Note that the serialized JSON output is not valid to be deserialized.
+        udf_python_warning = "Serialization of user-defined Python functions is not supported. " \
+                             "Any produced serialized JSON file output for this dataset pipeline is not valid " \
+                             "to be deserialized."
+        try:
+            if isinstance(self.transform, (FunctionType, MethodType)):
+                # common function type(include lambda, class method) / object method
+                logger.warning(udf_python_warning)
+                json_obj = {}
+                json_obj["tensor_op_name"] = self.transform.__name__
+                json_obj["python_module"] = self.__class__.__module__
+                return json.dumps(json_obj)
+            if callable(self.transform) and not isinstance(self.transform, (t.c_transforms.TensorOperation,
+                                                                            t.py_transforms.PyTensorOperation,
+                                                                            t.transforms.TensorOperation,
+                                                                            t.transforms.PyTensorOperation,
+                                                                            FuncWrapper)):
+                # udf callable class
+                logger.warning(udf_python_warning)
+                json_obj = {}
+                json_obj["tensor_op_name"] = type(self.transform).__name__
+                json_obj["python_module"] = self.__class__.__module__
+                return json.dumps(json_obj)
+            # dataset operations
+            return self.transform.to_json()
+        except Exception as e:
+            logger.warning("Skip user-defined Python method which cannot be serialized, reason is: " + str(e))
             json_obj = {}
-            json_obj["tensor_op_name"] = self.transform.__name__
-            json_obj["python_module"] = self.__class__.__module__
+            json_obj["tensor_op_name"] = "unknown"
+            json_obj["python_module"] = "unknown"
             return json.dumps(json_obj)
-        return self.transform.to_json()
+
+    def release_resource(self):
+        # release the executor which is used by current thread/process when
+        # use transform in eager mode in map op or batch op
+        # this will be call in MapOp::WorkerEntry and BatchOp::WorkerEntry
+        t.transforms.clean_unused_executors()

mindspore/dataset/transforms/transforms.py

@@ -17,6 +17,8 @@ The module transforms provides common operations, including Compose, OneHot and
 """
 import json
 from abc import ABC
+import os
+import threading
 
 import sys
 from enum import IntEnum
@@ -36,6 +38,28 @@ from ..core.datatypes import mstype_to_detype, nptype_to_detype
 from ..vision.py_transforms_util import is_pil
 
 
+# hold all the executor objects when in training procedure
+# key : pid_tid which distinguishes multiple executors by process_id + thread_id
+# value : executor object which lifecycle will always exist during training
+EXECUTORS_LIST = dict()
+
+
+# the follow case process / thread exit need call the function
+# 1. user defined dataset with process mode
+# 2. user defined dataset with thread mode
+# 3. user defined transform in map op with process mode
+# 4. user defined transform in map op with thread mode
+# 5. batch op with per_batch_map operation in process mode
+# 6. batch op with per_batch_map operation in thread mode
+def clean_unused_executors():
+    """
+    clean the unused executor object in UDF or map with PyFunc process / thread mode
+    """
+    key = str(os.getpid()) + "_" + str(threading.currentThread().ident)
+    if key in EXECUTORS_LIST:
+        EXECUTORS_LIST.pop(key)
+
+
 class TensorOperation:
     """
     Base class Tensor Ops
@@ -44,7 +68,7 @@ class TensorOperation:
     def __init__(self):
         super().__init__()
         self.implementation = None
-        self.callable_op_ = None
+        self.device_target = "CPU"
 
     def __call__(self, *input_tensor_list):
         """
@@ -62,9 +86,22 @@ class TensorOperation:
             except (RuntimeError, TypeError):
                 raise TypeError("Invalid user input. Got {}: {}, cannot be converted into tensor." \
                                 .format(type(tensor), tensor))
-        if not hasattr(self, 'callable_op_') or self.callable_op_ is None:
-            self.callable_op_ = cde.Execute(self.parse())
-        output_tensor_list = self.callable_op_(tensor_row)
+
+        # get or create the executor from EXECUTORS_LIST
+        executor = None
+        key = str(os.getpid()) + "_" + str(threading.currentThread().ident)
+        if key in EXECUTORS_LIST:
+            # get the executor by process id and thread id
+            executor = EXECUTORS_LIST[key]
+            # remove the old transform which in executor and update the new transform
+            executor.UpdateOperation(self.parse())
+        else:
+            # create a new executor by process id and thread_id
+            executor = cde.Execute(self.parse())
+            # add the executor the global EXECUTORS_LIST
+            EXECUTORS_LIST[key] = executor
+
+        output_tensor_list = executor(tensor_row)
         output_numpy_list = [x.as_array() for x in output_tensor_list]
         return output_numpy_list[0] if len(output_numpy_list) == 1 else tuple(output_numpy_list)
 
@@ -340,6 +377,20 @@ class Compose(CompoundOperation):
         """
         return util.compose(self.transforms, *args)
 
+    def __call__(self, *args):
+        '''
+        If PY op exists in self.transforms, should use _execute_py to keep the output types unchanged.
+        '''
+        if any([t.implementation == Implementation.PY for t in self.transforms]):
+            self.implementation = Implementation.PY
+        return super().__call__(*args)
+
+    def release_resource(self):
+        # release the executor which is used by current thread/process when
+        # use transform in eager mode in map op
+        # this will be call in MapOp::WorkerEntry
+        clean_unused_executors()
+
 
 class Concatenate(TensorOperation):
     """
@@ -476,7 +527,7 @@ class Mask(TensorOperation):
             ``Relational.GT``, ``Relational.LE``, ``Relational.GE``, take ``Relational.EQ`` as example,
             EQ refers to equal.
         constant (Union[str, int, float, bool]): Constant to be compared to.
-        dtype (mindspore.dtype, optional): Type of the generated mask. Default: ``mstype.bool\_``.
+        dtype (mindspore.dtype, optional): Type of the generated mask. Default: ``mstype.bool_``.
 
     Raises:
         TypeError: `operator` is not of type Relational.
@@ -519,20 +570,25 @@ class Mask(TensorOperation):
 
 class OneHot(TensorOperation):
     """
-    Tensor operation to apply one hot encoding.
+    Apply One-Hot encoding to the input labels.
+
+    For a 1-D input of shape :math:`(*)`, an output of shape :math:`(*, num_classes)` will be
+    returned, where the elements with index values equal to the input values will be set to 1,
+    and the rest will be set to 0. If a label smoothing rate is specified, the element values
+    are further smoothed to enhance generalization.
 
     Args:
-        num_classes (int): Number of classes of objects in dataset.
-            It should be larger than the largest label number in the dataset.
-        smoothing_rate (float, optional): Adjustable hyperparameter for label smoothing level.
-            Default: ``0.0``, means no smoothing is applied.
+        num_classes (int): Total number of classes. Must be greater than the maximum value
+            of the input labels.
+        smoothing_rate (float, optional): The amount of label smoothing. Must be between
+            [0.0, 1.0]. Default: ``0.0``, no label smoothing.
 
     Raises:
-        TypeError: `num_classes` is not of type int.
-        TypeError: `smoothing_rate` is not of type float or int.
-        ValueError: `smoothing_rate` is not in range [0.0, 1.0].
-        RuntimeError: Input tensor is not of type int.
-        RuntimeError: Input tensor is not a 1-D tensor.
+        TypeError: If `num_classes` is not of type int.
+        TypeError: If `smoothing_rate` is not of type float.
+        ValueError: If `smoothing_rate` is not in range of [0.0, 1.0].
+        RuntimeError: If input label is not of type int.
+        RuntimeError: If the dimension of the input label is not 1.
 
     Supported Platforms:
         ``CPU``
@@ -815,17 +871,16 @@ class RandomOrder(PyTensorOperation):
 
 class Relational(IntEnum):
     """
-    Relationship operator.
+    Relational operator.
 
-    Possible enumeration values are: ``Relational.EQ``, ``Relational.NE``, ``Relational.GT``, ``Relational.GE``,
-    ``Relational.LT``, ``Relational.LE``.
+    Available values are as follows:
 
-    - Relational.EQ: refers to Equality.
-    - Relational.NE: refers not equal, or Inequality.
-    - Relational.GT: refers to Greater than.
-    - Relational.GE: refers to Greater than or equal to.
-    - Relational.LT: refers to Less than.
-    - Relational.LE: refers to Less than or equal to.
+    - Relational.EQ: Equal to.
+    - Relational.NE: Not equal to.
+    - Relational.GT: Greater than.
+    - Relational.GE: Greater than or equal to.
+    - Relational.LT: Less than.
+    - Relational.LE: Less than or equal to.
     """
     EQ = 0
     NE = 1
@@ -871,25 +926,24 @@ class _SliceOption(cde.SliceOption):
 
 class Slice(TensorOperation):
     """
-    Slice operation to extract a tensor out using the given n slices.
+    Extract a slice from the input.
 
-    The functionality of Slice is similar to NumPy's indexing feature (Currently only rank-1 tensors are supported).
+    Currently, only 1-D input is supported.
 
     Args:
-        slices (Union[int, list[int], slice, None, Ellipsis]):
-            Maximum `n` number of arguments to slice a tensor of rank `n` .
-            One object in slices can be one of:
-
-            1.  :py:obj:`int`: Slice this index only along the first dimension. Negative index is supported.
-            2.  :py:obj:`list(int)`: Slice these indices along the first dimension. Negative indices are supported.
-            3.  :py:obj:`slice`: Slice the generated indices from the
-                `slice <https://docs.python.org/3.7/library/functions.html?highlight=slice#slice>`_ object along the
-                first dimension. Similar to start:stop:step.
-            4.  :py:obj:`None`: Slice the whole dimension. Similar to :py:obj:`[:]` in Python indexing.
-            5.  :py:obj:`Ellipsis`: Slice the whole dimension, same result with `None` .
+        slices (Union[int, list[int], slice, Ellipsis]): The desired slice.
+            If the input type is int, it will slice the element with the specified index value.
+            Negative index is also supported.
+            If the input type is list[int], it will slice all the elements with the specified index values.
+            Negative index is also supported.
+            If the input type is `slice <https://docs.python.org/3.7/library/functions.html#slice>`_ ,
+            it will slice according to its specified start position, stop position and step size.
+            If the input type is `Ellipsis <https://docs.python.org/3.7/library/constants.html#Ellipsis>`_ ,
+            all elements will be sliced.
+            If the input is None, all elements will be sliced.
 
     Raises:
-        TypeError: If `slices` is not of type int, list[int], :py:obj:`slice` , :py:obj:`None` or :py:obj:`Ellipsis` .
+        TypeError: If `slices` is not of type Union[int, list[int], slice, Ellipsis].
 
     Supported Platforms:
         ``CPU``
@@ -930,7 +984,8 @@ class TypeCast(TensorOperation):
     Tensor operation to cast to a given MindSpore data type or NumPy data type.
 
     Note:
-        This operation supports running on Ascend or GPU platforms by Offload.
+        This operation is executed on the CPU by default, but it is also supported
+        to be executed on the GPU or Ascend via heterogeneous acceleration.
 
     Args:
         data_type (Union[mindspore.dtype, numpy.dtype]): mindspore.dtype or numpy.dtype (e.g. `numpy.float32`)
@@ -940,7 +995,7 @@ class TypeCast(TensorOperation):
         TypeError: If `data_type` is not of MindSpore data type bool, int, float, string or type :class:`numpy.dtype` .
 
     Supported Platforms:
-        ``Ascend`` ``GPU`` ``CPU``
+        ``CPU`` ``GPU`` ``Ascend``
 
     Examples:
         >>> import mindspore.dataset as ds

mindspore/dataset/utils/browse_dataset.py

@@ -48,15 +48,21 @@ def imshow_det_bbox(image, bboxes, labels, segm=None, class_names=None, score_th
         font_size (int, float): Font size of texts. Default: ``0.8``.
         show (bool): Whether to show the image. Default: ``True``.
         win_name (str): The window name. Default: ``"win"``.
-        wait_time (int): Value of waitKey param. Default: ``2000``, means display interval is 2000ms.
+        wait_time (int): Waiting time delay for a key event in milliseconds. During image display,
+            if there is no key pressed, wait for this time then jump to next image. If ESC is pressed,
+            quit display immediately. If any other key is pressed, stop waiting then jump to
+            next image directly. Default: ``2000`` , wait 2000ms then jump to next image.
         out_file (str, optional): The filename to write the imagee. Default: ``None``. File extension name
             is required to indicate the image compression type, e.g. 'jpg', 'png'.
 
     Returns:
         ndarray, The image with bboxes drawn on it.
 
+    Note:
+        This interface relies on the `opencv-python` library.
+
     Raises:
-        ImportError: If opencv-python is not installed.
+        ImportError: If `opencv-python` is not installed.
         AssertionError: If `image` is not in (H, W, C) or (C, H, W) format.
         AssertionError: If `bboxes` is not in (N, 4) or (N, 5) format.
         AssertionError: If `labels` is not in (N, 1) format.

mindspore/dataset/utils/line_reader.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Efficient line based file reading.
+"""Efficient line-based file reading.
 """
 import os
 
@@ -22,28 +22,26 @@ from ..core.validator_helpers import check_filename, check_uint64, check_value
 
 class LineReader:
     """
-    Efficient file line reader.
+    Line-based file reader.
 
-    This class is used to hold the line offsets of line based file.
-
-    The following functionality is provided:
-
-    - len(): return the number of lines in the file
-    - readline(line): open file handle (if not opened yet), and read a line in the file
-    - close(): close the file handle opened in readline
+    Cache the line-based meta data of the file in advance to achieve random-access reading of each file line.
 
     Args:
-        filename (str): line based file to be loaded.
+        filename (str): Filename to be read.
 
     Raises:
-        TypeError: Parameter `filename` is wrong.
-        RuntimeError: The input file does not exist or is not a regular file.
+        TypeError: If `filename` is not of type int.
+        RuntimeError: If `filename` does not exist or is not a regular file.
 
     Examples:
         >>> from mindspore.dataset import LineReader
         >>>
         >>> reader = LineReader("/path/to/txt/or/csv/file")
+        >>> # Read the first line of csv file
         >>> reader.readline(1)
+        >>> # Return the row size in csv file
+        >>> reader.len()
+        >>> # Close the handle
         >>> reader.close()
     """
 
@@ -84,22 +82,22 @@ class LineReader:
         self.close()
 
     def len(self):
-        """Get the total number of lines in the file"""
+        """Get the total number of lines in the current file."""
         return len(self.offsets)
 
     def readline(self, line):
         """
-        Read specified line content.
+        Reads the contents of the specified line.
 
         Args:
-            line (int): the line number to be read, starting at 1.
+            line (int): The line number to be read, with a starting line number of 1.
 
         Returns:
-            str, line content (until line break character).
+            str, the contents of the corresponding line, without line break characters.
 
         Raises:
-            TypeError: Parameter `line` is the wrong type.
-            ValueError: Parameter `line` exceeds the file range.
+            TypeError: If `line` is not of type int.
+            ValueError: If `line` exceeds the total number of lines in the file.
         """
         check_uint64(line, "line")
         check_value(line, [1, len(self.offsets)], "line")
@@ -119,7 +117,7 @@ class LineReader:
         return content
 
     def close(self):
-        """Close the file"""
+        """Close the file handle."""
         if self.fo_handle is None:
             return
         self.fo_handle.close()

mindspore/dataset/vision/__init__.py

@@ -32,7 +32,7 @@ Note: Legacy c_transforms and py_transforms are deprecated but can still be impo
     import mindspore.dataset.vision.py_transforms as py_vision
 
 See `Vision Transforms
-<https://www.mindspore.cn/tutorials/en/r2.1/beginner/transforms.html#vision-transforms>`_ tutorial for more details.
+<https://www.mindspore.cn/tutorials/en/r2.2/beginner/transforms.html#vision-transforms>`_ tutorial for more details.
 
 Descriptions of common data processing terms are as follows:
 
@@ -43,10 +43,10 @@ Descriptions of common data processing terms are as follows:
 The data transform operation can be executed in the data processing pipeline or in the eager mode:
 
 - Pipeline mode is generally used to process big datasets. Examples refer to
-  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.1/api_python/
+  `introduction to data processing pipeline <https://www.mindspore.cn/docs/en/r2.2/api_python/
   mindspore.dataset.html#introduction-to-data-processing-pipeline>`_ .
 - Eager mode is more like a function call to process data. Examples refer to
-  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/r2.1/advanced/dataset/eager.html>`_ .
+  `Lightweight Data Processing <https://www.mindspore.cn/tutorials/en/r2.2/advanced/dataset/eager.html>`_ .
 """
 from . import c_transforms
 from . import py_transforms

mindspore/dataset/vision/c_transforms.py

@@ -610,7 +610,8 @@ class Decode(ImageTensorOperation):
         return super().__call__(img)
 
     def parse(self):
-        return cde.DecodeOperation(self.rgb)
+        # deprecated api just support cpu device target
+        return cde.DecodeOperation(self.rgb, "CPU")
 
 
 class Equalize(ImageTensorOperation):
@@ -842,7 +843,8 @@ class Normalize(ImageTensorOperation):
         self.std = std
 
     def parse(self):
-        return cde.NormalizeOperation(self.mean, self.std, True)
+        # deprecated api just support cpu device target
+        return cde.NormalizeOperation(self.mean, self.std, True, "CPU")
 
 
 class NormalizePad(ImageTensorOperation):
@@ -2275,7 +2277,8 @@ class Resize(ImageTensorOperation):
         self.interpolation = interpolation
 
     def parse(self):
-        return cde.ResizeOperation(self.size, DE_C_INTER_MODE.get(self.interpolation))
+        # deprecated api just support cpu device target
+        return cde.ResizeOperation(self.size, DE_C_INTER_MODE.get(self.interpolation), "CPU")
 
 
 class ResizeWithBBox(ImageTensorOperation):