mindspore 2.0.0a0__cp37-none-any.whl → 2.0.0rc1__cp37-none-any.whl

mindspore/{parallel/nn/layers.py → dataset/debug/__init__.py}

@@ -1,4 +1,4 @@
-# Copyright 2021 Huawei Technologies Co., Ltd
+# Copyright 2023 Huawei Technologies Co., Ltd
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,12 +11,11 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ============================================================================
-"""
-The basic layer of the Transformer Networks. This is an experimental interface that is subject to
-change or deletion.
-"""
+# ==============================================================================
+"""Init file for dataset pipeline debug mode"""
 from __future__ import absolute_import
-from mindspore.nn.transformer.layers import FixedSparseAttention
 
-__all__ = ["FixedSparseAttention"]
+from mindspore.dataset.debug.debug_hook import DebugHook
+from mindspore.dataset.debug.pre_defined_hook import PrintMetaDataHook, PrintDataHook
+
+__all__ = ["DebugHook", "PrintMetaDataHook", "PrintDataHook"]

mindspore/dataset/debug/debug_hook.py

@@ -0,0 +1,65 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+This module defines the class for minddata pipeline debugger.
+class DebugHook is not exposed to users as an external API.
+"""
+
+from abc import ABC, abstractmethod
+from mindspore import log as logger
+
+
+class DebugHook(ABC):
+    """
+    The base class for Dataset Pipeline Python Debugger hook. All user defined hook behaviors
+    must inherit this base class.
+
+    To debug the input and output data of map operation in dataset pipeline, users can add
+    breakpoint to or single stepping in this class. They can also see the type and shape of
+    the data from the log being printed.
+
+    Args:
+        prev_op_name (str, optional): name of the operation before current debugging point.
+    """
+    def __init__(self, prev_op_name=None):
+        self.prev_op_name = prev_op_name
+
+    def __call__(self, *args):
+        # log op name
+        if self.prev_op_name:
+            log_message = "Debugging the output of the operation [{}].".format(self.prev_op_name)
+        else:
+            log_message = "Debugging the input of the first operation."
+        logger.info(log_message)
+
+        ######################## NOTE ########################
+        # Add a breakpoint to the following line to inspect
+        # input and output of each transform.
+        ######################################################
+        self.compute(args)
+        return args
+
+    @abstractmethod
+    def compute(self, *args):
+        """
+        Defines the debug behaviour to be performed. This method must be overridden by all subclasses.
+        """
+        raise RuntimeError("compute() is not overridden in subclass of class DebugHook.")
+
+    def set_previous_op_name(self, prev_op_name):
+        """
+        Set prev_op_name.
+        """
+        self.prev_op_name = prev_op_name

mindspore/dataset/debug/pre_defined_hook.py

@@ -0,0 +1,67 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""
+This module defines the subclass of DebugHook for minddata pipeline debugger.
+All these class are pre-defined for users for basic debugging purpose.
+"""
+
+import collections
+import numpy as np
+from PIL import Image
+from mindspore import log as logger
+from mindspore.dataset.debug.debug_hook import DebugHook
+
+
+class PrintMetaDataHook(DebugHook):
+    """
+    Debug hook used for MindData debug mode to print type and shape of data.
+    """
+    def __init__(self):
+        super().__init__()
+
+    def compute(self, *args):
+        for col_idx, col in enumerate(*args):
+            log_message = "Column {}. ".format(col_idx)
+            # log type
+            log_message += "The type is [{}].".format(type(col))
+
+            # log shape/size
+            if isinstance(col, np.ndarray):
+                log_message += " The shape is [{}].".format(col.shape)
+            elif isinstance(col, Image.Image):
+                log_message += " The shape is [{}].".format(col.size)
+            elif isinstance(col, collections.abc.Sized):
+                log_message += " The size is [{}].".format(len(col))
+            logger.info(log_message)
+        return args
+
+
+class PrintDataHook(DebugHook):
+    """
+    Debug hook used for MindData debug mode to print data.
+    """
+    def __init__(self):
+        super().__init__()
+
+    def compute(self, *args):
+        for col_idx, col in enumerate(*args):
+            log_message = "Column {}. ".format(col_idx)
+            if isinstance(col, Image.Image):
+                data = np.asarray(col)
+                log_message += "The data is [{}].".format(data)
+            else:
+                log_message += "The data is [{}].".format(col)
+            logger.info(log_message)
+        return args

mindspore/dataset/engine/__init__.py

@@ -50,6 +50,7 @@ __all__ = ["Caltech101Dataset",        # Vision
            "FashionMnistDataset",      # Vision
            "FlickrDataset",            # Vision
            "Flowers102Dataset",        # Vision
+           "Food101Dataset",           # Vision
            "ImageFolderDataset",       # Vision
            "KITTIDataset",             # Vision
            "KMnistDataset",            # Vision
@@ -62,10 +63,12 @@ __all__ = ["Caltech101Dataset",        # Vision
            "Places365Dataset",         # Vision
            "QMnistDataset",            # Vision
            "RandomDataset",            # Vision
+           "RenderedSST2Dataset",      # Vision
            "SBDataset",                # Vision
            "SBUDataset",               # Vision
            "SemeionDataset",           # Vision
            "STL10Dataset",             # Vision
+           "SUN397Dataset",            # Vision
            "SVHNDataset",              # Vision
            "USPSDataset",              # Vision
            "VOCDataset",               # Vision
@@ -83,6 +86,7 @@ __all__ = ["Caltech101Dataset",        # Vision
            "PennTreebankDataset",      # Text
            "SogouNewsDataset",         # Text
            "SQuADDataset",             # Text
+           "SST2Dataset",              # Text
            "TextFileDataset",          # Text
            "UDPOSDataset",             # Text
            "WikiTextDataset",          # Text
@@ -102,10 +106,10 @@ __all__ = ["Caltech101Dataset",        # Vision
            "GeneratorDataset",         # User Defined
            "NumpySlicesDataset",       # User Defined
            "PaddedDataset",            # User Defined
-           "GraphData",                # Graph Data
+           "GraphData",                # Graph
            "Graph",                    # Graph
-           "InMemoryGraphDataset",     # InMemoryGraphDataset
-           "ArgoverseDataset",         # ArgoverseDataset
+           "InMemoryGraphDataset",     # Graph
+           "ArgoverseDataset",         # Graph
            "DistributedSampler",       # Sampler
            "RandomSampler",            # Sampler
            "SequentialSampler",        # Sampler

mindspore/dataset/engine/cache_client.py

@@ -27,7 +27,7 @@ class DatasetCache:
     A client to interface with tensor caching service.
 
     For details, please check `Tutorial <https://www.mindspore.cn/
-    tutorials/experts/en/r2.0.0-alpha/dataset/cache.html>`_ .
+    tutorials/experts/en/r2.0/dataset/cache.html>`_ .
 
     Args:
         session_id (int): A user assigned session id for the current pipeline.