mindspore 2.2.11__cp39-cp39-win_amd64.whl → 2.3.0__cp39-cp39-win_amd64.whl

mindspore/rewrite/{symbol_tree_builder.py → symbol_tree/symbol_tree_builder.py}

@@ -16,12 +16,12 @@
 from typing import Optional
 import ast
 import inspect
+from textwrap import dedent
 
 from mindspore.nn import Cell
 from .symbol_tree import SymbolTree
-from .parsers.parser_register import ParserRegister
-from .parsers.parser import Parser
-from .ast_transformers import FlattenRecursiveStmt
+from ..parsers import Parser, ParserRegister
+from ..ast_helpers import AstFlattener
 
 
 class SymbolTreeBuilder:
@@ -33,14 +33,14 @@ class SymbolTreeBuilder:
     """
 
     # Entry function of the forward computation process
-    entry_function = "construct"
+    entry_functions = ["construct"]
 
     def __init__(self, network: Cell):
         if not isinstance(network, Cell):
-            raise RuntimeError("Only support network with Cell type now, ", network)
+            raise TypeError("Type of network should be Cell, but got ", network)
         self._origin_net = network
         network_str = inspect.getsource(type(network))
-        self._ast_root: ast.Module = ast.parse(network_str)
+        self._ast_root: ast.Module = ast.parse(dedent(network_str))
         self._root_tree: Optional[SymbolTree] = None
 
     @staticmethod
@@ -54,7 +54,7 @@ class SymbolTreeBuilder:
         Returns:
              An instance of ast been optimized.
         """
-        ast_root = FlattenRecursiveStmt().transform(ast_root, [SymbolTreeBuilder.entry_function])
+        ast_root = AstFlattener().transform(ast_root, SymbolTreeBuilder.entry_functions)
         return ast_root
 
     def build(self) -> SymbolTree:
@@ -67,7 +67,7 @@ class SymbolTreeBuilder:
 
         self._ast_root = SymbolTreeBuilder.ast_transform(self._ast_root)
         if not isinstance(self._ast_root, ast.Module):
-            raise RuntimeError("ast_root should be a ast.Module")
+            raise TypeError("Type of ast_root should be ast.Module, but got ", self._ast_root)
         self._root_tree: SymbolTree = SymbolTree(self._origin_net, self._ast_root)
         parser: Parser = ParserRegister.instance().get_parser(ast.Module)
         parser.process(self._root_tree, self._ast_root, None)

mindspore/rewrite/{symbol_tree_dumper.py → symbol_tree/symbol_tree_dumper.py}

@@ -15,9 +15,9 @@
 """SymbolTree dumper."""
 import inspect
 
-from .node.node import Node
-from .api.node_type import NodeType
-from .api.scoped_value import ScopedValue, ValueType
+from ..node import Node
+from ..api.node_type import NodeType
+from ..api.scoped_value import ScopedValue, ValueType
 
 
 class SymbolTreeDumper:
@@ -45,7 +45,7 @@ class SymbolTreeDumper:
 
     def _dump_inputs(self):
         """Dump inputs of SymbolTree."""
-        inputs = self._symbol_tree.get_inputs()
+        inputs = self._symbol_tree.get_input_nodes()
         self._dump_buffer += f"#Inputs num     : {len(inputs)}\n"
         for single_input in inputs:
             targets = single_input.get_targets()

mindspore/run_check/_check_version.py

@@ -259,7 +259,7 @@ class AscendEnvChecker(EnvChecker):
 
     def __init__(self, library_path):
         self.library_path = library_path
-        self.version = ["7.1"]
+        self.version = ["7.2", "7.3"]
         atlas_nnae_version = "/usr/local/Ascend/nnae/latest/compiler/version.info"
         atlas_toolkit_version = "/usr/local/Ascend/ascend-toolkit/latest/compiler/version.info"
         hisi_fwk_version = "/usr/local/Ascend/latest/compiler/version.info"
@@ -400,11 +400,12 @@ class AscendEnvChecker(EnvChecker):
         curr_path = os.path.abspath(os.path.dirname(__file__))
         cust_aicpu_path = os.path.abspath(os.path.join(curr_path, "../lib/plugin/ascend/custom_aicpu_ops"))
         cust_aicore_path = os.path.abspath(os.path.join(curr_path, "../lib/plugin/ascend/custom_aicore_ops"))
+        cust_ascendc_path = os.path.abspath(os.path.join(curr_path, "../lib/plugin/ascend/custom_ascendc_ops"))
         if os.getenv('ASCEND_CUSTOM_OPP_PATH'):
             os.environ['ASCEND_CUSTOM_OPP_PATH'] = os.environ['ASCEND_CUSTOM_OPP_PATH'] + ":" + \
-                                                   cust_aicore_path + ":" + cust_aicpu_path
+                                                   cust_ascendc_path + ":" + cust_aicore_path + ":" + cust_aicpu_path
         else:
-            os.environ['ASCEND_CUSTOM_OPP_PATH'] = cust_aicore_path + ":" + cust_aicpu_path
+            os.environ['ASCEND_CUSTOM_OPP_PATH'] = cust_ascendc_path + ":" + cust_aicore_path + ":" + cust_aicpu_path
         plugin_dir = os.path.dirname(self.library_path)
         akg_dir = os.path.join(plugin_dir, "ascend")
         AscendEnvChecker._concat_variable('LD_LIBRARY_PATH', akg_dir)
@@ -502,13 +503,6 @@ def check_env(device, _):
 
 def set_env(device, library_path):
     """callback function for setting environment variables"""
-    if not os.getenv("MS_DEV_CLOSE_VERSION_CHECK") is None:
-        if device in os.environ["MS_DEV_CLOSE_VERSION_CHECK"]:
-            return
-        os.environ["MS_DEV_CLOSE_VERSION_CHECK"] = os.environ["MS_DEV_CLOSE_VERSION_CHECK"] + ":" + device
-    else:
-        os.environ["MS_DEV_CLOSE_VERSION_CHECK"] = device
-
     if device.lower() == "ascend":
         env_checker = AscendEnvChecker(library_path)
     elif device.lower() == "gpu":
@@ -533,9 +527,7 @@ def check_version_and_env_config():
         except OSError:
             logger.warning("Pre-Load Library libgomp.so.1 failed, which might cause TLS memory allocation failure. If "
                            "the failure occurs, please refer to the FAQ for a solution: "
-                           "https://www.mindspore.cn/docs/en/r2.2/faq/installation.html.")
-        if not os.getenv("MS_DEV_CLOSE_VERSION_CHECK") is None:
-            return
+                           "https://www.mindspore.cn/docs/en/master/faq/installation.html.")
         MSContext.get_instance().register_check_env_callback(check_env)
         MSContext.get_instance().register_set_env_callback(set_env)
         MSContext.get_instance().set_device_target_inner(MSContext.get_instance().get_param(ms_ctx_param.device_target))
@@ -577,6 +569,6 @@ def _add_cuda_path():
                 os.environ['PATH'] += os.pathsep + cuda_home_bin_path
 
 
-check_version_and_env_config()
 _set_pb_env()
+check_version_and_env_config()
 _add_cuda_path()

mindspore/run_check/run_check.py

@@ -39,7 +39,7 @@ def _check_mul():
     input_x = ms.Tensor(np.array([1.0, 2.0, 3.0]), ms.float32)
     input_y = ms.Tensor(np.array([4.0, 5.0, 6.0]), ms.float32)
     mul = ms.ops.Mul()
-    mul(input_x, input_y)
+    mul(input_x, input_y).asnumpy()
     print("The result of multiplication calculation is correct, MindSpore has been installed on platform "
           f"[{ms.get_context('device_target')}] successfully!")
 

mindspore/safeguard/rewrite_obfuscation.py

@@ -22,9 +22,9 @@ from mindspore import ops, nn
 from mindspore.common.tensor import Tensor
 from mindspore import log as logger
 from mindspore import load_checkpoint, save_checkpoint
-from mindspore.rewrite import SymbolTree, Node, NodeType, TreeNodeHelper, ScopedValue
-from mindspore.rewrite.parsers.class_def_parser import ClassDefParser
-from mindspore.rewrite.parsers.class_def_parser import ModuleParser
+from mindspore.rewrite import SymbolTree, Node, NodeType, ScopedValue
+from mindspore.rewrite.parsers import ClassDefParser
+from mindspore.rewrite.parsers import ModuleParser
 
 OBF_RATIOS_LENGTH = 1
 MAX_OBF_RATIOS_NUM = 50
@@ -73,7 +73,7 @@ def obfuscate_ckpt(network, ckpt_files, target_modules=None, saved_path='./', ob
 
     Examples:
         >>> from mindspore import obfuscate_ckpt, save_checkpoint
-        >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # Refer to https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> save_checkpoint(net, './test_net.ckpt')
         >>> target_modules = ['', 'fc1|fc2']
@@ -204,7 +204,7 @@ def load_obf_params_into_net(network, target_modules, obf_ratios, data_parallel_
         >>> from mindspore import obfuscate_ckpt, save_checkpoint, load_checkpoint, Tensor
         >>> import mindspore.common.dtype as mstype
         >>> import numpy as np
-        >>> # Refer to https://gitee.com/mindspore/docs/blob/r2.2/docs/mindspore/code/lenet.py
+        >>> # Refer to https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/lenet.py
         >>> net = LeNet5()
         >>> save_checkpoint(net, './test_net.ckpt')
         >>> target_modules = ['', 'fc1|fc2']
@@ -476,22 +476,12 @@ def _obfuscate_network(model, path_list, target_list, data_parallel_num=1, **kwa
                         _insert_mul(stree, node, OBF_RATIOS_INSERT_INDEX)
                         OBF_RATIOS_INSERT_INDEX += 1
 
-    def _update_subnet(stree: SymbolTree, substree: SymbolTree, subnode: Node):
+    def _update_subnet(substree: SymbolTree, subnode: Node):
         """update the network once the subnet is obfuscated"""
-        new_net = substree.get_network()
         input_y_node = substree.get_node("input_y_obf")
         if input_y_node is None:
             return
-        arg_list = subnode.get_args().copy()
-        kwargs_list = list(subnode.get_kwargs().values())
-        arg_list.extend(kwargs_list)
-        v: str = input_y_node.get_targets()[0].value
-        arg_obf: ScopedValue = ScopedValue.create_naming_value("y_obf=" + v)
-        arg_list.append(arg_obf)
-        target_list = subnode.get_targets().copy()
-        name = subnode.get_name()
-        new_node = subnode.create_call_cell(cell=new_net, targets=target_list, args=arg_list, name=name)
-        stree.replace(subnode, [new_node])
+        subnode.get_handler().append_kwarg({"y_obf": input_y_node.get_targets()[0]})
 
     def _traverse(stree, i=0):
         """traverse and obfuscate the original network"""
@@ -500,11 +490,11 @@ def _obfuscate_network(model, path_list, target_list, data_parallel_num=1, **kwa
         for node in stree.nodes():
             node_name = node.get_name()
             if node.get_node_type() == NodeType.Tree and node_name.startswith(path_list[i]):
-                sub_stree = TreeNodeHelper.get_sub_tree(node)
+                sub_stree = node.get_sub_tree()
                 _traverse(sub_stree, i + 1)
                 _insert_input(sub_stree, arg_name='y_obf')
                 _insert_mul_by_name(sub_stree, after_name_list=target_list[i + 1])
-                _update_subnet(stree, sub_stree, node)
+                _update_subnet(sub_stree, node)
 
     def _register_denied_func_decorators(fn):
         """set the function decorators which should be denied for parse"""

mindspore/train/__init__.py

@@ -26,20 +26,21 @@ from mindspore.train.amp import build_train_network
 from mindspore.train.loss_scale_manager import LossScaleManager, FixedLossScaleManager, DynamicLossScaleManager
 from mindspore.train.serialization import save_checkpoint, load_checkpoint, load_param_into_net, export, \
     load, parse_print, build_searched_strategy, merge_sliced_parameter, load_distributed_checkpoint, \
-    async_ckpt_thread_status, restore_group_info_list, convert_model, obfuscate_model, export_split_mindir
+    async_ckpt_thread_status, restore_group_info_list, convert_model, obfuscate_model, export_split_mindir, \
+    load_checkpoint_async, check_checkpoint
 from mindspore.train.callback import Callback, LossMonitor, TimeMonitor, ModelCheckpoint, SummaryCollector, \
-    CheckpointConfig, RunContext, LearningRateScheduler, SummaryLandscape, \
-    History, LambdaCallback, ReduceLROnPlateau, EarlyStopping, OnRequestExit, BackupAndRestore
+    CheckpointConfig, RunContext, LearningRateScheduler, SummaryLandscape, FlopsUtilizationCollector, \
+    History, LambdaCallback, ReduceLROnPlateau, EarlyStopping, OnRequestExit, BackupAndRestore, MindIOTTPAdapter
 from mindspore.train.summary import SummaryRecord
 from mindspore.train.train_thor import ConvertNetUtils, ConvertModelUtils
 from mindspore.train.metrics import *
 from mindspore.train.data_sink import data_sink
 
 __all__ = ["Model", "DatasetHelper", "connect_network_with_dataset", "build_train_network", "LossScaleManager",
-           "FixedLossScaleManager", "DynamicLossScaleManager", "save_checkpoint", "load_checkpoint",
+           "FixedLossScaleManager", "DynamicLossScaleManager", "save_checkpoint", "load_checkpoint", "check_checkpoint",
            "load_param_into_net", "export", "load", "export_split_mindir", "parse_print", "build_searched_strategy",
            "merge_sliced_parameter", "load_distributed_checkpoint", "async_ckpt_thread_status",
-           "restore_group_info_list", "convert_model", "data_sink", "obfuscate_model"]
+           "restore_group_info_list", "convert_model", "data_sink", "obfuscate_model", "load_checkpoint_async"]
 __all__.extend(callback.__all__)
 __all__.extend(summary.__all__)
 __all__.extend(train_thor.__all__)

mindspore/train/_utils.py

@@ -16,7 +16,9 @@
 from __future__ import absolute_import
 
 import os
+import json
 from collections.abc import Iterable
+
 import numpy as np
 
 from mindspore.common.tensor import Tensor
@@ -154,11 +156,11 @@ def _check_to_numpy(plugin, tensor, prim=None):
     if plugin == 'image':
         if np_value.ndim == 4:
             return np_value
-        raise ValueError(f'For "{summary_name}", The tensor seems not to hold a valid image.')
+        raise ValueError(f'For "{summary_name}", the tensor seems not to hold a valid image.')
     if plugin in ('tensor', 'histogram'):
         if np_value.ndim > 0:
             return np_value
-        raise ValueError(f'For "{summary_name}", The value should not be empty.')
+        raise ValueError(f'For "{summary_name}", the value should not be empty.')
     return np_value
 
 
@@ -173,9 +175,9 @@ def check_summary_param(summary_name, tag, tensor):
                             f'but got {type(tensor).__name__}')
         _check_to_numpy(plugin, tensor, prim=True)
     except TypeError as err:
-        raise TypeError(err)
+        raise TypeError(err) from err
     except ValueError as err:
-        raise ValueError(err)
+        raise ValueError(err) from err
     finally:
         debug_ops.SUMMARY_TENSOR_CACHE = []
 
@@ -263,3 +265,175 @@ def read_proto(file_name, proto_format="MINDIR", display_data=False):
                     ele.tensor_content = b'\0'
 
     return model
+
+
+def parse_strategy_ckpt(file_name):
+    """
+    Parses a strategy ckpt layout file and returns the rank location dict.
+
+    Args:
+        file_name (str):Strategy ckpt file name.
+
+    Returns:
+        Dict, layout dict. Key is parameter name, value is (dev_matrix, tensor_map).
+
+    Examples:
+        >>> from mindspore.train.utils import parse_strategy_ckpt
+        >>> layout_dict = parse_strategy_ckpt("/path/to/strategy.ckpt")
+        {"param1": [[4, 4], [0, -1]], "param2": [[4, 4], [-1, 0]],,,,}
+    """
+    model = ckpt_strategy()
+    with open(file_name, "rb") as f:
+        pb_content = f.read()
+        model.ParseFromString(pb_content)
+    layout_dict = {}
+    for param in model.parallel_layout_item:
+        dev_matrix = []
+        tensor_map = []
+        for ele in param.parallel_layouts.dev_matrix[0].ListFields()[0][1]:
+            dev_matrix.append(ele)
+
+        for ele in param.parallel_layouts.tensor_map[0].ListFields()[0][1]:
+            tensor_map.append(ele)
+        layout_dict[param.param_name] = [dev_matrix, tensor_map]
+    return layout_dict
+
+
+def get_parameter_redundancy(layout_obj, initial_rank=0):
+    """
+    Get parameter redundancy map.
+
+    Args:
+        layout_obj (Union[str, layout): File name of `strategy.ckpt` or net.parameter_layout_dict.
+        initial_rank (int): Start rank id for each pipeline. Default: 0.
+
+    Returns:
+        Dict, dict of parameter redundancy info.
+
+    Examples:
+        >>> from mindspore.train.utils import get_parameter_redundancy
+        >>> param_redundancy_dict = get_parameter_redundancy("/path/to/strategy.ckpt")
+        {'param1': ((0, 1, 2, 3, 4, 5, 6, 7),),
+         'param2': ((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)),
+         'param3': ((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15)),
+         'param4': ((0, 4, 8, 12), (1, 5, 9, 13), (2, 6, 10, 14), (3, 7, 11, 15))}
+    """
+    if isinstance(layout_obj, str):
+        parameter_layout = parse_strategy_ckpt(layout_obj)
+    else:
+        parameter_layout = {}
+        for k, v in layout_obj.items():
+            parameter_layout[k] = v[:2]
+
+    param_redundancy_dict = {}
+    for key, (slices, deploy_loc, *_) in parameter_layout.items():
+        redundancy_matrix = np.zeros(shape=slices + [len(slices)], dtype=np.int8)
+        for i in deploy_loc:
+            internal_slice = tuple(slice(None) for _ in range(i))
+            for j in range(slices[-i - 1]):
+                if i == -1:
+                    continue
+                else:
+                    redundancy_matrix[(..., j) + internal_slice + (i,)] = j
+        locate_list = redundancy_matrix.reshape((-1, len(slices))).tolist()
+        redundancy_dict = {}
+        for index, locate in enumerate(locate_list):
+            redundancy_dict.setdefault(tuple(locate), []).append(index+initial_rank)
+        redundancy_list = []
+        for _, indices in sorted(redundancy_dict.items()):
+            redundancy_list.append(tuple(indices))
+
+        param_redundancy_dict[key] = tuple(redundancy_list)
+    return param_redundancy_dict
+
+
+def _collect_settings_by_rank(redundancy_map):
+    """
+    Collect parameter redundancy map by rank id.
+
+    {"param1":((1,3,5,7),(2,4,6,8)),"param2":((1,3,5,7),(2,4,6,8))}
+    ->{(1,3,5,7):{"param1", "param2"},(2,4,6,8):{"param1", "param2"}}
+    """
+    redundancy_map_reversed = {}
+    for key, redundancy in redundancy_map.items():
+        for index, item in enumerate(redundancy):
+            redundancy_map_reversed.setdefault(item, []).append(
+                (key, index))
+    return redundancy_map_reversed
+
+
+def _restructure(input_dict):
+    """
+    Flatten and reorganize the nested dictionary structure."""
+    if all(not isinstance(item, tuple) for item in input_dict):
+        return input_dict
+    res_dict = {}
+    for key, values in input_dict.items():
+        for index, value in enumerate(values):
+            res_dict.setdefault(key[index % len(key)], []).append(value)
+    return _restructure(res_dict)
+
+
+def _rotate_list_elements(i, input_list):
+    """Rotate element list."""
+    rotated_list = [input_list[(i + j) % len(input_list)] for j in
+                    range(len(input_list))]
+    return rotated_list
+
+
+def remove_param_redundancy(param_redundancy_dict, keep_redundancy=1):
+    """
+    Remove parameter redundancy, get the single parameter for each rank id.
+    Args:
+        param_redundancy_dict (Dict): Parameter redundancy dict.
+        keep_redundancy (Int): Keep redundancy number.
+
+    Returns:
+        Dict, single parameter for each rank id. Key is rank_id, value is set(params).
+
+    Examples:
+        >>> from mindspore.train.utils import get_parameter_redundancy, remove_param_redundancy
+        >>> param_redundancy_dict = get_parameter_redundancy("/path/to/strategy.ckpt")
+        >>> single_parameter = remove_param_redundancy(param_redundancy_dict)
+        {0: {param1, param3}, 1: {param2, param4},,,}}
+    """
+    redundancy_dict_reversed = _collect_settings_by_rank(param_redundancy_dict)
+    sorted_layouts = {}
+    for device_layout, layer_names_list in redundancy_dict_reversed.items():
+        sorted_layer_names = [item[0] for item in layer_names_list]
+        sorted_layouts[device_layout] = sorted_layer_names
+    result = {}
+    for i in range(keep_redundancy):
+        rotated_layouts = {tuple(_rotate_list_elements(i, key)): value for
+                           key, value in sorted_layouts.items()}
+        restructured_layouts = _restructure(rotated_layouts)
+        for key, value in restructured_layouts.items():
+            result.setdefault(key, set()).update(set(value))
+    return result
+
+
+def parse_hccl_file(hccl_file_path):
+    """
+    Parses an HCCL configuration JSON file, return a dict key is rank_id, value is device_ip.
+
+    Args:
+        hccl_file_path (str): The path to the HCCL configuration JSON file.
+
+    Returns:
+        Dict: A Dict, key is rank_id, value is device_ip.
+
+    Examples:
+        >>> from mindspore.train.utils import parse_hccl_file
+        >>> rankid_dict = parse_hccl_file("/path/to/hccl.json")
+        {0: "10.11.10.163", 1: "10.11.10.164", 2: "10.11.10.165", 3: "10.11.10.166",,,,}
+    """
+    with open(hccl_file_path) as f:
+        hccl_dict = json.load(f)
+    server_list = hccl_dict["server_list"]
+    rankid_dict = {}
+    for server in server_list:
+        device_list = server["device"]
+        for device in device_list:
+            rankid_dict[int(device["rank_id"])] = device["device_ip"]
+
+    return rankid_dict