mindspore 2.0.0rc1__cp38-cp38-manylinux1_x86_64.whl → 2.2.0__cp38-cp38-manylinux1_x86_64.whl

mindspore/dataset/core/validator_helpers.py

@@ -20,7 +20,6 @@ from __future__ import absolute_import
 import inspect
 from multiprocessing import cpu_count
 import os
-import numpy as np
 
 import mindspore._c_dataengine as cde
 from mindspore import log as logger
@@ -56,7 +55,7 @@ def is_iterable(obj):
         obj (any): object to check if iterable
 
     Returns:
-        bool, true if object iteratable
+        bool, true if object iterable
     """
     try:
         iter(obj)
@@ -690,67 +689,6 @@ def validate_dataset_param_value(param_list, param_dict, param_type):
                 type_check(param_dict.get(param_name), (param_type,), param_name)
 
 
-def check_gnn_list_of_pair_or_ndarray(param, param_name):
-    """
-    Check if the input parameter is a list of tuple or numpy.ndarray.
-
-    Args:
-        param (Union[list[tuple], nd.ndarray]): param.
-        param_name (str): param_name.
-
-    Returns:
-        Exception: TypeError if error.
-    """
-    type_check(param, (list, np.ndarray), param_name)
-    if isinstance(param, list):
-        param_names = ["node_list[{0}]".format(i) for i in range(len(param))]
-        type_check_list(param, (tuple,), param_names)
-        for idx, pair in enumerate(param):
-            if not len(pair) == 2:
-                raise ValueError("Each member in {0} must be a pair which means length == 2. Got length {1}".format(
-                    param_names[idx], len(pair)))
-            column_names = ["node_list[{0}], number #{1} element".format(idx, i + 1) for i in range(len(pair))]
-            type_check_list(pair, (int,), column_names)
-    elif isinstance(param, np.ndarray):
-        if param.ndim != 2:
-            raise ValueError("Input ndarray must be in dimension 2. Got {0}".format(param.ndim))
-        if param.shape[1] != 2:
-            raise ValueError("Each member in {0} must be a pair which means length == 2. Got length {1}".format(
-                param_name, param.shape[1]))
-        if not param.dtype == np.int32:
-            raise TypeError("Each member in {0} should be of type int32. Got {1}.".format(
-                param_name, param.dtype))
-
-
-def check_gnn_list_or_ndarray(param, param_name, data_type=int):
-    """
-    Check if the input parameter is list or numpy.ndarray.
-
-    Args:
-        param (Union[list, nd.ndarray]): param.
-        param_name (str): param_name.
-        data_type(object): data type.
-
-    Returns:
-        Exception: TypeError if error.
-    """
-
-    type_check(param, (list, np.ndarray), param_name)
-    if isinstance(param, list):
-        param_names = ["param_{0}".format(i) for i in range(len(param))]
-        type_check_list(param, (data_type,), param_names)
-
-    elif isinstance(param, np.ndarray):
-        if data_type == int:
-            data_type = np.int32
-        elif data_type == str:
-            data_type = np.str_
-
-        if param.dtype != data_type:
-            raise TypeError("Each member in {0} should be of type {1}. Got {2}.".format(
-                param_name, data_type, param.dtype))
-
-
 def check_tensor_op(param, param_name):
     """check whether param is a tensor op or a callable Python function"""
     if not isinstance(param, cde.TensorOp) and not callable(param) and not getattr(param, 'parse', None):

mindspore/dataset/debug/debug_hook.py

@@ -18,7 +18,6 @@ class DebugHook is not exposed to users as an external API.
 """
 
 from abc import ABC, abstractmethod
-from mindspore import log as logger
 
 
 class DebugHook(ABC):
@@ -26,23 +25,50 @@ class DebugHook(ABC):
     The base class for Dataset Pipeline Python Debugger hook. All user defined hook behaviors
     must inherit this base class.
 
-    To debug the input and output data of map operation in dataset pipeline, users can add
-    breakpoint to or single stepping in this class. They can also see the type and shape of
-    the data from the log being printed.
+    To debug the input and output data of `map` operation in dataset pipeline, users can add
+    breakpoint in `compute` method, or print types and shapes of the data.
 
     Args:
-        prev_op_name (str, optional): name of the operation before current debugging point.
+        prev_op_name (str, optional): name of the operation before current debugging point. Default: ``None``.
+
+    Examples:
+        >>> import mindspore.dataset as ds
+        >>> import mindspore.dataset.debug as debug
+        >>>
+        >>> class CustomizedHook(debug.DebugHook):
+        ...     def __init__(self):
+        ...         super().__init__()
+        ...
+        ...     def compute(self, *args):
+        ...         import pdb
+        ...         pdb.set_trace()
+        ...         print("Data after decode", *args)
+        ...         return args
+        >>>
+        >>> # Enable debug mode
+        >>> ds.config.set_debug_mode(True, debug_hook_list=[CustomizedHook()])
+        >>>
+        >>> # Define dataset pipeline
+        >>> dataset = ds.ImageFolderDataset(dataset_dir="/path/to/image_folder_dataset_directory")
+        >>> # Insert debug hook after `Decode` operation.
+        >>> dataset = dataset.map([vision.Decode(), CustomizedHook(), vision.CenterCrop(100)])
     """
     def __init__(self, prev_op_name=None):
         self.prev_op_name = prev_op_name
+        self.is_first_op = None
 
     def __call__(self, *args):
-        # log op name
-        if self.prev_op_name:
-            log_message = "Debugging the output of the operation [{}].".format(self.prev_op_name)
+        # If insert debug function into map, like [Decode(), debug_fun(), Resize],
+        # the debug_fun does not have self.prev_op_name, so skip the common print.
+        if not self.prev_op_name:
+            pass
         else:
-            log_message = "Debugging the input of the first operation."
-        logger.info(log_message)
+            # log op name
+            if self.is_first_op:
+                log_message = "[Dataset debugger] Print the [INPUT] of the operation [{}].".format(self.prev_op_name)
+            else:
+                log_message = "[Dataset debugger] Print the [OUTPUT] of the operation [{}].".format(self.prev_op_name)
+            print(log_message, flush=True)
 
         ######################## NOTE ########################
         # Add a breakpoint to the following line to inspect
@@ -55,11 +81,17 @@ class DebugHook(ABC):
     def compute(self, *args):
         """
         Defines the debug behaviour to be performed. This method must be overridden by all subclasses.
+        Refers to the example above to define a customized hook.
+
+        Args:
+            *args (Any): The input/output of the operation, just print it.
         """
         raise RuntimeError("compute() is not overridden in subclass of class DebugHook.")
 
     def set_previous_op_name(self, prev_op_name):
-        """
-        Set prev_op_name.
-        """
+        # Set prev_op_name.
         self.prev_op_name = prev_op_name
+
+    def set_is_first(self, is_first_op):
+        # Set op is the first in map.
+        self.is_first_op = is_first_op

mindspore/dataset/debug/pre_defined_hook.py

@@ -20,7 +20,6 @@ All these class are pre-defined for users for basic debugging purpose.
 import collections
 import numpy as np
 from PIL import Image
-from mindspore import log as logger
 from mindspore.dataset.debug.debug_hook import DebugHook
 
 
@@ -34,17 +33,18 @@ class PrintMetaDataHook(DebugHook):
     def compute(self, *args):
         for col_idx, col in enumerate(*args):
             log_message = "Column {}. ".format(col_idx)
-            # log type
-            log_message += "The type is [{}].".format(type(col))
 
             # log shape/size
             if isinstance(col, np.ndarray):
+                log_message += "The dtype is [{}].".format(col.dtype)
                 log_message += " The shape is [{}].".format(col.shape)
             elif isinstance(col, Image.Image):
+                log_message += "The type is [{}].".format(type(col))
                 log_message += " The shape is [{}].".format(col.size)
             elif isinstance(col, collections.abc.Sized):
+                log_message += "The type is [{}].".format(type(col))
                 log_message += " The size is [{}].".format(len(col))
-            logger.info(log_message)
+            print(log_message, flush=True)
         return args
 
 
@@ -63,5 +63,5 @@ class PrintDataHook(DebugHook):
                 log_message += "The data is [{}].".format(data)
             else:
                 log_message += "The data is [{}].".format(col)
-            logger.info(log_message)
+            print(log_message, flush=True)
         return args

mindspore/dataset/engine/__init__.py

@@ -31,7 +31,6 @@ from .datasets_text import *
 from .datasets_audio import *
 from .datasets_standard_format import *
 from .datasets_user_defined import *
-from .graphdata import GraphData, Graph, InMemoryGraphDataset, ArgoverseDataset, SamplingStrategy, OutputFormat
 from .iterators import *
 from .obs.obs_mindrecord_dataset import *
 from .samplers import *
@@ -106,10 +105,6 @@ __all__ = ["Caltech101Dataset",        # Vision
            "GeneratorDataset",         # User Defined
            "NumpySlicesDataset",       # User Defined
            "PaddedDataset",            # User Defined
-           "GraphData",                # Graph
-           "Graph",                    # Graph
-           "InMemoryGraphDataset",     # Graph
-           "ArgoverseDataset",         # Graph
            "DistributedSampler",       # Sampler
            "RandomSampler",            # Sampler
            "SequentialSampler",        # Sampler

mindspore/dataset/engine/cache_client.py

@@ -27,28 +27,30 @@ class DatasetCache:
     A client to interface with tensor caching service.
 
     For details, please check `Tutorial <https://www.mindspore.cn/
-    tutorials/experts/en/r2.0/dataset/cache.html>`_ .
+    tutorials/experts/en/r2.2/dataset/cache.html>`_ .
 
     Args:
         session_id (int): A user assigned session id for the current pipeline.
-        size (int, optional): Size of the memory set aside for the row caching. Default: 0, which means unlimited,
+        size (int, optional): Size of the memory set aside for the row caching. Default: ``0``, which means unlimited,
             note that it might bring in the risk of running out of memory on the machine.
-        spilling (bool, optional): Whether or not spilling to disk if out of memory. Default: False.
-        hostname (str, optional): Host name. Default: None, use default hostname '127.0.0.1'.
-        port (int, optional): Port to connect to server. Default: None, use default port 50052.
-        num_connections (int, optional): Number of tcp/ip connections. Default: None, use default value 12.
+        spilling (bool, optional): Whether or not spilling to disk if out of memory. Default: ``False``.
+        hostname (str, optional): Host name. Default: ``None`` , use default hostname '127.0.0.1'.
+        port (int, optional): Port to connect to server. Default: ``None`` , use default port 50052.
+        num_connections (int, optional): Number of tcp/ip connections. Default: ``None`` , use default value 12.
         prefetch_size (int, optional): The size of the cache queue between operations.
-            Default: None, use default value 20.
+            Default: ``None`` , use default value 20.
 
     Examples:
-            >>> import mindspore.dataset as ds
-            >>>
-            >>> # Create a cache instance, in which session_id is generated from command line `cache_admin -g`
-            >>> # In the following code, suppose the session_id is 780643335
-            >>> some_cache = ds.DatasetCache(session_id=780643335, size=0)
-            >>>
-            >>> dataset_dir = "/path/to/image_folder_dataset_directory"
-            >>> ds1 = ds.ImageFolderDataset(dataset_dir, cache=some_cache)
+        >>> import subprocess
+        >>> import mindspore.dataset as ds
+        >>>
+        >>> # Create a cache instance with command line `cache_admin --start` and create a session with `cache_admin -g`
+        >>> # After creating cache with a valid session, get session id with command `cache_admin --list_sessions`
+        >>> session_id = subprocess.getoutput('cache_admin --list_sessions | tail -1 | awk -F " " \'{{print $1;}}\'')
+        >>> some_cache = ds.DatasetCache(session_id=int(session_id), size=0)
+        >>>
+        >>> dataset_dir = "/path/to/image_folder_dataset_directory"
+        >>> dataset = ds.ImageFolderDataset(dataset_dir, cache=some_cache)
     """
 
     def __init__(self, session_id, size=0, spilling=False, hostname=None, port=None, num_connections=None,
@@ -83,6 +85,27 @@ class DatasetCache:
         Get the statistics from a cache. After data pipeline, three types of statistics can be obtained,
         including average number of cache hits (avg_cache_sz), number of caches in memory (num_mem_cached)
         and number of caches in disk (num_disk_cached).
+
+        Examples:
+            >>> import os
+            >>> import mindspore.dataset as ds
+            >>>
+            >>> # In example above, we created cache with a valid session id
+            >>> id = int(os.popen('cache_admin --list_sessions | tail -1 | awk -F " " \'{{print $1;}}\'').read())
+            >>> some_cache = ds.DatasetCache(session_id=id, size=0)
+            >>>
+            >>> # run the dataset pipeline to trigger cache
+            >>> dataset = ds.ImageFolderDataset("/path/to/image_folder_dataset_directory", cache=some_cache)
+            >>> data = list(dataset)
+            >>>
+            >>> # get status of cache
+            >>> stat = some_cache.get_stat()
+            >>> # Average cache size
+            >>> cache_sz = stat.avg_cache_sz
+            >>> # Number of rows cached in memory
+            >>> num_mem_cached = stat.num_mem_cached
+            >>> # Number of rows spilled to disk
+            >>> num_disk_cached = stat.num_disk_cached
         """
         return self.cache_client.GetStat()