mindspore 2.3.0__cp310-cp310-win_amd64.whl

mindspore/parallel/mpi/_mpi_config.py

@@ -0,0 +1,116 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+The MPI config, used to configure the MPI environment.
+"""
+import threading
+from mindspore._c_expression import MpiConfig
+from mindspore._checkparam import args_type_check
+
+
+class _MpiConfig:
+    """
+    _MpiConfig is the config tool for controlling MPI
+
+    Note:
+        Create a config through instantiating MpiConfig object is not recommended.
+        should use MpiConfig() to get the config since MpiConfig is singleton.
+    """
+    _instance = None
+    _instance_lock = threading.Lock()
+
+    def __init__(self):
+        self._mpiconfig_handle = MpiConfig.get_instance()
+
+    def __new__(cls, *args, **kwargs):
+        if cls._instance is None:
+            cls._instance_lock.acquire()
+            cls._instance = object.__new__(cls)
+            cls._instance_lock.release()
+        return cls._instance
+
+    def __getattribute__(self, attr):
+        value = object.__getattribute__(self, attr)
+        if attr == "_mpiconfig_handle" and value is None:
+            raise ValueError("mpiconfig handle is none in MpiConfig!!!")
+        return value
+
+    @property
+    def enable_mpi(self):
+        """Get enable mpi."""
+        return self._mpiconfig_handle.get_enable_mpi()
+
+    @enable_mpi.setter
+    def enable_mpi(self, enable_mpi):
+        self._mpiconfig_handle.set_enable_mpi(enable_mpi)
+
+_k_mpi_config = None
+
+
+def _mpi_config():
+    """
+    Get the global mpi config, if mpi config is not created, create a new one.
+
+    Returns:
+        _MpiConfig, the global mpi config.
+    """
+    global _k_mpi_config
+    if _k_mpi_config is None:
+        _k_mpi_config = _MpiConfig()
+    return _k_mpi_config
+
+
+@args_type_check(enable_mpi=bool)
+def _set_mpi_config(**kwargs):
+    """
+    Sets mpi config for running environment.
+
+    mpi config should be configured before running your program. If there is no configuration,
+    mpi module will be disabled by default.
+
+    Note:
+        Attribute name is required for setting attributes.
+
+    Args:
+        enable_mpi (bool): Whether to enable mpi. Default: False.
+
+    Raises:
+        ValueError: If input key is not an attribute in mpi config.
+
+    Examples:
+        >>> mpiconfig.set_mpi_config(enable_mpi=True)
+    """
+    for key, value in kwargs.items():
+        if not hasattr(_mpi_config(), key):
+            raise ValueError("Set mpi config keyword %s is not recognized!" % key)
+        setattr(_mpi_config(), key, value)
+
+
+def _get_mpi_config(attr_key):
+    """
+    Gets mpi config attribute value according to the input key.
+
+    Args:
+        attr_key (str): The key of the attribute.
+
+    Returns:
+        Object, The value of given attribute key.
+
+    Raises:
+        ValueError: If input key is not an attribute in config.
+    """
+    if not hasattr(_mpi_config(), attr_key):
+        raise ValueError("Get context keyword %s is not recognized!" % attr_key)
+    return getattr(_mpi_config(), attr_key)

mindspore/parallel/parameter_broadcast.py

@@ -0,0 +1,152 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""Parameter broadcast"""
+from __future__ import absolute_import
+
+__all__ = ["parameter_broadcast"]
+
+import numpy as np
+
+
+def parameter_broadcast(net, layout, cur_rank=0, initial_rank=0):
+    """
+    Broadcast parameter to other rank in data parallel dimension.
+
+    .. warning::
+        This is an experimental API that is subject to change or deletion.
+
+    Args:
+        net (Cell): The network where the parameters will be broadcasted.
+        layout (Dict): Parameter layout dictionary. Come from
+            :func:`mindspore.nn.Cell.parameter_layout_dict`
+            or read from file(for example: "strategy.ckpt" saved by using the
+            `strategy_ckpt_config` parameter of :func:`mindspore.set_auto_parallel_context`).
+            The key is param name, the value is the layout of this parameter.
+        cur_rank (int, optional): current rank id. Default: ``0``.
+        initial_rank (int, optional): Start rank id for each pipeline. Default: ``0``.
+
+    Raises:
+        ValueError: `cur_rank` is not rank id of current rank.
+        ValueError: `initial_rank` is not the start rank id of current pipeline stage.
+        ValueError: Parameter name in `layout` can not be found in
+            :func:`mindspore.nn.Cell.parameters_dict`.
+
+    Examples:
+        >>> import os
+        >>> import mindspore as ms
+        >>> import mindspore.dataset as ds
+        >>> from mindspore import nn, ops
+        >>> from mindspore.communication import init
+        >>> from mindspore.common.initializer import initializer
+        >>> from mindspore.train import Model
+        >>> from mindspore.parallel.parameter_broadcast import parameter_broadcast
+        >>> from mindspore.train.serialization import load_checkpoint, load_param_into_net
+        >>> ms.set_context(mode=ms.GRAPH_MODE)
+        >>> ms.set_context(max_device_memory="28GB")
+        >>> ms.set_auto_parallel_context(parallel_mode=ms.ParallelMode.SEMI_AUTO_PARALLEL)
+        >>> init()
+        >>> ms.set_seed(1)
+        >>> class Network(nn.Cell):
+        ...     def __init__(self):
+        ...         super().__init__()
+        ...         self.flatten = ops.Flatten()
+        ...         self.fc1_weight = ms.Parameter(initializer("normal", [28*28, 512], ms.float32))
+        ...         self.fc2_weight = ms.Parameter(initializer("normal", [512, 512], ms.float32))
+        ...         self.fc3_weight = ms.Parameter(initializer("normal", [512, 10], ms.float32))
+        ...         self.matmul1 = ops.MatMul()
+        ...         self.relu1 = ops.ReLU()
+        ...         self.matmul2 = ops.MatMul()
+        ...         self.relu2 = ops.ReLU()
+        ...         self.matmul3 = ops.MatMul()
+        ...     def construct(self, x):
+        ...         x = self.flatten(x)
+        ...         x = self.matmul1(x, self.fc1_weight)
+        ...         x = self.relu1(x)
+        ...         x = self.matmul2(x, self.fc2_weight)
+        ...         x = self.relu2(x)
+        ...         logits = self.matmul3(x, self.fc3_weight)
+        ...         return logits
+        >>> net = Network()
+        >>> net.matmul1.shard(((2, 4), (4, 1)))
+        >>> net.relu1.shard(((4, 1),))
+        >>> net.matmul2.shard(((1, 8), (8, 1)))
+        >>> net.relu2.shard(((8, 1),))
+        >>> # Create the dataset taking MNIST as an example. Refer to
+        >>> # https://gitee.com/mindspore/docs/blob/master/docs/mindspore/code/mnist.py
+        >>> dataset = create_dataset()
+        >>> optim = nn.SGD(net.trainable_params(), 1e-2)
+        >>> loss = nn.CrossEntropyLoss()
+        >>> model = Model(net, loss_fn=loss, optimizer=optim)
+        >>> model.train(1, dataset)
+        >>> ms.save_checkpoint(net, "./simple.ckpt", False)
+        >>> layout = model.train_network.parameter_layout_dict
+        >>> param_dict = load_checkpoint("./simple.ckpt")
+        >>> load_param_into_net(net, param_dict)
+        >>> rank_id = os.environ["RANK_ID"]
+        >>> parameter_broadcast(model.train_network, layout, int(rank_id), 0)
+        >>> class LossCallBack(Callback):
+        ...     def step_end(self, run_context):
+        ...         cb_params = run_context.original_args()
+        ...         print("step end, cur step num: ", cb_params.cur_step_num, flush=True)
+        >>> model.train(1, dataset, callbacks=[LossCallBack()])
+    """
+    if not layout:
+        return
+    import mindspore as ms
+    from mindspore import Tensor
+    from mindspore.communication import get_rank, create_group, get_group_size
+    from mindspore.train._utils import get_parameter_redundancy, remove_param_redundancy
+    from mindspore.nn.wrap.cell_wrapper import AllreduceGraph
+    origin_parallel_mode = ms.get_auto_parallel_context("parallel_mode")
+    if origin_parallel_mode not in ("semi_auto_parallel", "auto_parallel"):
+        return
+    if cur_rank != get_rank():
+        raise ValueError(f"For parameter broadcast, the cur_rank: {cur_rank} is wrong.")
+    if initial_rank % (get_group_size() / ms.get_auto_parallel_context("pipeline_stages")) != 0:
+        raise ValueError(f"For parameter broadcast, the initial_rank: {initial_rank} is wrong.")
+    param_redundancy = get_parameter_redundancy(layout, initial_rank)
+    if not param_redundancy:
+        return
+    single_params = remove_param_redundancy(param_redundancy)
+    if not single_params:
+        return
+    param_redundancy_reversed = {}
+    for key, redundancy in param_redundancy.items():
+        for item in redundancy:
+            if len(item) == 1:
+                continue
+            if cur_rank in item:
+                param_redundancy_reversed.setdefault(item, []).append(key)
+    if not param_redundancy_reversed:
+        return
+    if cur_rank not in single_params:
+        return
+    net_param_dict = net.parameters_dict()
+    ms.set_auto_parallel_context(parallel_mode="hybrid_parallel")
+    for group, params in param_redundancy_reversed.items():
+        create_group(str(group), list(group))
+        allreduce_input = []
+        for param in params:
+            if param not in net_param_dict:
+                raise ValueError(f"For parameter broadcast, the param: {param} can not be found.")
+            real_param = net_param_dict[param]
+            if param not in single_params[cur_rank]:
+                real_param.set_data(Tensor(np.zeros(real_param.shape), dtype=real_param.dtype))
+            allreduce_input.append(real_param)
+        if not allreduce_input:
+            continue
+        allreduce_graph = AllreduceGraph(allreduce_input, str(group))
+        allreduce_graph()
+    ms.set_auto_parallel_context(parallel_mode=origin_parallel_mode)

mindspore/parallel/shard.py

@@ -0,0 +1,350 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""shard"""
+
+import copy
+import mindspore as ms
+from mindspore import log as logger
+from mindspore._c_expression import Shard_
+
+
+class Layout():
+    """
+    Parallel layout describes the detailed sharding information.
+
+    Note:
+        - It is valid only in semi auto parallel or auto parallel mode.
+        - The multiplication result of the `device_matrix` must be equal to the device count in a pipeline stage.
+        - When the layout function is invoked to constructs a sharding strategy, each alias name is only allowed to be
+          used once to shard a tensor.
+
+    Args:
+        device_matrix (tuple): Describe the shape of devices arrangement, its element type is int.
+        alias_name (tuple): The alias name for each axis of device_matrix, its length shoits element type is string.
+                            When using "interleaved_parallel" as an alias name, the tensor would be split into multiple
+                            copies on the corresponding partition dimension on a single card. The corresponding value
+                            of "interleaved_parallel" in device_matrix must be 2.
+
+    Raises:
+        TypeError: `device_matrix` is not a tuple type.
+        TypeError: `alias_name` is not a tuple type.
+        ValueError: `device_matrix` length is not equal to `alias_name` length.
+        TypeError: The element of `device_matrix` is not int type.
+        TypeError: The element of `alias_name` is not a str type.
+        ValueError: The element of `alias_name` is an empty str.
+        ValueError: The element of `alias_name` is "None".
+        ValueError: `alias_name` contains repeated element.
+
+    Examples:
+        >>> from mindspore import Layout
+        >>> layout = Layout((2, 2, 2), ("dp", "sp", "mp"))
+        >>> layout0 = layout("dp", "mp")
+        >>> print(layout0.to_dict())
+        {"device_matrix": (2, 2, 2), "tensor_map": (2, 0)}
+        >>> # Total device num is 4, but split the tensor in local device into two copies.
+        >>> layout = Layout((2, 2, 2), ("dp", "sp", "interleaved_parallel"))
+        >>> layout1 = layout(("dp", "interleaved_parallel"), "sp")
+    """
+
+    def __init__(self, device_matrix, alias_name):
+        if not isinstance(device_matrix, tuple):
+            raise TypeError(f'device_matrix must be tuple type, but got:{type(device_matrix)}')
+        if not isinstance(alias_name, tuple):
+            raise TypeError(f'alias_name must be tuple type, but got:{type(alias_name)}')
+        if len(device_matrix) != len(alias_name):
+            raise ValueError(f'device_matrix length should be equal to alias_name length')
+        for in_ele in device_matrix:
+            if not isinstance(in_ele, int):
+                raise TypeError(f'The element of device_matrix must be int type, but got:{type(in_ele)}')
+        for in_ele in alias_name:
+            if not isinstance(in_ele, str):
+                raise TypeError(f'The element of alias_name must be str type, but got:{type(in_ele)}')
+            if not in_ele:
+                raise ValueError(f"The element of alias_name can not be empty.")
+            if in_ele == "None":
+                raise ValueError(f"The element of alias_name can not set 'None', because 'None' means no sharding.")
+        if len(set(alias_name)) != len(alias_name):
+            raise ValueError(f'Each element of alias_name {alias_name} should be different')
+        inter_key = "interleaved_parallel"
+        if inter_key in alias_name and alias_name.index(inter_key) != len(alias_name) - 1:
+            raise ValueError(f"When alias_name {alias_name} contains keyword 'interleaved_parallel',"
+                             f" it should be at the last dim of alias_name, which means the virtual sharding.")
+        if inter_key in alias_name and device_matrix[alias_name.index(inter_key)] != 2:
+            raise ValueError(f"When alias_name {alias_name} contains keyword 'interleaved_parallel',"
+                             f" the corresponding dim of device_matrix should be 2.")
+        self._device_shape = device_matrix
+        self._alias_name = alias_name
+        self._tensor_map = None
+
+    def __call__(self, *tensor_map):
+        self._tensor_map = ()
+        writed_map = ()
+        for ele in tensor_map:
+            if isinstance(ele, tuple):
+                ele_map = ()
+                for item in ele:
+                    if item == "None":
+                        ele_map += (-1,)
+                        continue
+                    if item not in self._alias_name:
+                        raise ValueError(f'The axis {item} is not found in {self._alias_name}')
+                    if item in writed_map:
+                        raise ValueError(f'The axis {item} has been set more than one in {self._alias_name}')
+                    ele_map += (len(self._alias_name) - 1 - self._alias_name.index(item),)
+                    writed_map += (item,)
+                self._tensor_map += (ele_map,)
+                continue
+            if ele == "None":
+                self._tensor_map += (-1,)
+                continue
+            if ele not in self._alias_name:
+                raise ValueError(f'The axis {ele} is not found in {self._alias_name}')
+            if ele in writed_map:
+                raise ValueError(f'The axis {ele} has been set more than one in {self._alias_name}')
+            self._tensor_map += (len(self._alias_name) - 1 - self._alias_name.index(ele),)
+            writed_map += (ele,)
+        return copy.deepcopy(self)
+
+    def to_dict(self):
+        """
+        Transform layout to a dictionary.
+        """
+        if self._device_shape is None:
+            raise ValueError("The device_shape of layout is None")
+        if self._tensor_map is None:
+            raise ValueError("The tensor_map of layout is None")
+        interleaved_parallel = "interleaved_parallel" in self._alias_name
+        return {"device_matrix": self._device_shape, "tensor_map": self._tensor_map,
+                "interleaved_parallel": interleaved_parallel}
+
+
+
+class Shard(Shard_):
+    """Shard operation"""
+
+    def __init__(self):
+        """Initialize Shard."""
+        super().__init__('Shard')
+        self.shard_fn = None
+        self.fn = None
+        self.in_strategy = None
+        self.out_strategy = None
+        self.parameter_plan = None
+        self.device = None
+        self.level = None
+
+    def __call__(self, fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascend", level=0):
+        parallel_mode = ms.context.get_auto_parallel_context("parallel_mode")
+        if parallel_mode not in ["auto_parallel", "semi_auto_parallel"]:
+            raise AssertionError(
+                f"Cell shard only supports auto parallel and semi auto parallel.")
+        if ms.context.get_context("device_target") not in ["Ascend", "GPU"]:
+            raise AssertionError(
+                f"'Shard' now only supports 'Ascend' and 'GPU'")
+        if parallel_mode == "auto_parallel" and \
+            ms.context.get_auto_parallel_context("search_mode") != "sharding_propagation":
+            raise AssertionError(f"'search_mode' must be 'sharding_propagation' for 'Shard' when the "
+                                 f"'parallel_mode' is 'auto_parallel.'")
+        if not isinstance(in_strategy, tuple):
+            raise TypeError(
+                f"For 'Shard', the 'in_strategy' should be a tuple, but got {type(in_strategy).__name__}")
+        if not isinstance(out_strategy, (type(None), tuple)):
+            raise TypeError(f"For 'Shard', the 'out_strategy' should be None or tuple, "
+                            f"but got {type(out_strategy).__name__}")
+
+        if not isinstance(device, str):
+            raise TypeError(f"For 'Shard', the 'device' should be a string, "
+                            f"but got {type(device).__name__}")
+        if not isinstance(level, int):
+            raise TypeError(f"For 'Shard', the 'level' should be an integer, "
+                            f"but got {type(level).__name__}")
+
+        if ms.get_algo_parameters("fully_use_devices") is True:
+            logger.warning("After calling 'shard', the environment variable 'fully_use_devices' "
+                           "will be overwritten as False.")
+            ms.set_algo_parameters(fully_use_devices=False)
+
+        if ms.context.get_auto_parallel_context("full_batch_is_set") is False:
+            logger.warning("When calling the shard interface, "
+                           "'dataset_strategy' or 'full_batch' is not manually set by the user, "
+                           "and the 'dataset_strategy' will be set to 'full_batch'.")
+            ms.context.set_auto_parallel_context(dataset_strategy="full_batch")
+
+        if self._is_attrs_has_been_set(fn, in_strategy, out_strategy, device, level):
+            return self.shard_fn
+        shard_ = Shard()
+
+        if isinstance(fn, ms.nn.Cell):
+            for param in fn.trainable_params():
+                param.is_in_shard = True
+
+        # Set parameter layout to corresponding parameter
+        self._set_param_layout_into_parameter(fn, parameter_plan)
+
+        def shard_fn(*args):
+            @ms.common.jit(hash_args=fn)
+            def after_shard(*args):
+                return shard_(fn, in_strategy, out_strategy, device, level)(*args)
+
+            return after_shard(*args)
+
+        self.shard_fn = shard_fn
+        self.fn = fn
+        self.in_strategy = in_strategy
+        self.out_strategy = out_strategy
+        self.device = device
+        self.level = level
+        return self.shard_fn
+
+    @staticmethod
+    def _search_parameter_by_name(param_name: str, net):
+        param_name = param_name.replace("self.", "")
+        for param in net.trainable_params():
+            if param.name == param_name:
+                return param
+        return None
+
+    @staticmethod
+    def _check_layout_is_valid(param_name, param_shape, param_strategy):
+        if len(param_strategy) != len(param_shape):
+            raise ValueError(f"For {param_name}, the length of param_strategy: {len(param_strategy)}, "
+                             f"is not equal to param_shape len: {len(param_shape)}.")
+        for i, _ in enumerate(param_strategy):
+            if param_shape[i] % param_strategy[i] != 0:
+                raise ValueError(f"For '{param_name}', the param_shape is {param_shape} and "
+                                 f"the setting param_strategy is {param_strategy}. "
+                                 f"The param_shape[{i}]: {param_shape[i]} cannot be divisible by "
+                                 f"param_strategy[{i}]: {param_strategy[i]}.")
+
+    def _set_param_layout_into_parameter(self, fn, parameter_plan):
+        """ Set param_strategy into parameter if fn is a Cell and parameter_plan is a dict."""
+        if parameter_plan is None:
+            return
+        if isinstance(parameter_plan, dict):
+            if not isinstance(fn, ms.nn.Cell):
+                raise TypeError(
+                    f"If parameter_plan is set, type of fn must be mindspore.nn.Cell, but got {type(fn)}")
+            for k in parameter_plan.keys():
+                v = parameter_plan[k]
+                if not isinstance(k, str) or not isinstance(v, tuple):
+                    raise TypeError(f"For 'Shard', the type of each key and value in 'parameter_plan' must be str and "
+                                    f"tuple, but got {type(k).__name__} and {type(v).__name__}")
+        else:
+            raise TypeError(f"For 'Shard', the 'parameter_plan' should be a dict or None, "
+                            f"but got {type(parameter_plan).__name__}")
+
+        for param_name in parameter_plan.keys():
+            param_strategy = parameter_plan[param_name]
+            param = self._search_parameter_by_name(param_name, fn)
+            if param is None:
+                logger.warning(
+                    f"{param_name} is not exist, ignored its setting.")
+                continue
+
+            self._check_layout_is_valid(
+                param_name, param.shape, param_strategy)
+            if param.param_info.param_strategy:
+                logger.warning(f"The layout of parameter '{param_name}' "
+                               f"has been set to {param.param_info.param_strategy}, "
+                               f"current setting {param_strategy} will be ignored.")
+            param.param_info.param_strategy = param_strategy
+
+    def _is_attrs_has_been_set(self, fn, in_strategy, out_strategy, device, level):
+        return self.shard_fn is not None and self.fn == fn and self.in_strategy == in_strategy and \
+            self.out_strategy == out_strategy and self.device == device and self.level == level
+
+
+def shard(fn, in_strategy, out_strategy=None, parameter_plan=None, device="Ascend", level=0):
+    """
+    Defining the input and output layouts of this cell and the parallel strategies of remaining ops will be
+    generated by sharding propagation. In PyNative mode, use this method to specify a Cell for distributed
+    execution in graph mode. In Graph mode, use this method to specify distribution strategy for a Cell,
+    strategy for others will be set by sharding propagation.
+    in_strategy and out_strategy define the input and output layout respectively.
+    in_strategy/out_strategy should be a tuple, each element of which corresponds to the desired layout of
+    this input/output, and None represents data_parallel,
+    which can refer to the description of :func:`mindspore.ops.Primitive.shard`.
+    The parallel strategies of remaining operators are derived from the strategy specified by the input and output.
+
+    Note:
+        If ms.shard is called, the parallel mode in `set_auto_parallel_context` (parallel_mode) will be set to
+        "auto_parallel" and the search mode (search_mode) to "sharding_propagation".
+        If the input contain Parameter, its strategy should be set in `in_strategy`.
+
+    Args:
+        fn (Union[Cell, Function]): Function to be executed in parallel.
+                                    Its arguments and return value must be Tensor or Parameter.
+                                    If `fn` is a Cell with parameters, `fn` needs to be an instantiated object,
+                                    otherwise its arguments cannot be accessed.
+        in_strategy (tuple): Define the layout of inputs, each element of the tuple should be a tuple or None.
+                             Tuple defines the layout of the corresponding input
+                             and None represents a data parallel strategy.
+        out_strategy (Union[tuple, None]): Define the layout of outputs similar with `in_strategy`.
+                                           It is not in use right now. Default: ``None`` .
+        parameter_plan (Union[dict, None]): Define the layout for the specified parameters. Each element in dict
+                                            defines the layout of the parameter like "param_name: layout".
+                                            The key is a parameter name of type 'str'.
+                                            The value is a 1-D integer tuple, indicating the corresponding layout.
+                                            If the parameter name is incorrect or the corresponding parameter
+                                            has been set, the parameter setting will be ignored.
+                                            Default: ``None`` .
+        device (string): Select a certain `device` target. It is not in use right now.
+                         Support ["CPU", "GPU", "Ascend"]. Default: ``"Ascend"`` .
+        level (int): Option for parallel strategy infer algorithm, namely the object function, maximize computation
+                     over communication ratio, maximize speed performance, minimize memory usage etc. It is not in
+                     use right now. Support [0, 1, 2]. Default: ``0`` .
+
+    Returns:
+        Function, return the function that will be executed under auto parallel process.
+
+    Raises:
+        AssertionError: If parallel mode is not "auto_parallel" nor "semi_auto_parallel".
+        AssertionError: If device_target it not "Ascend" or "GPU".
+        TypeError: If `in_strategy` is not a tuple.
+        TypeError: If `out_strategy` is not a tuple or None.
+        TypeError: If `parameter_plan` is not a dict or None.
+        TypeError: If any key in `parameter_plan` is not a str.
+        TypeError: If any value in `parameter_plan` is not a tuple.
+        TypeError: If `device` is not a str.
+        TypeError: If `level` is not an integer.
+
+    Supported Platforms:
+        ``Ascend`` ``GPU``
+
+    Examples:
+        >>> import numpy as np
+        >>> import mindspore as ms
+        >>> from mindspore import Tensor
+        >>> from mindspore.communication import init
+        >>> ms.set_context(mode=ms.PYNATIVE_MODE)
+        >>> init()
+        >>> ms.set_auto_parallel_context(parallel_mode="auto_parallel", search_mode="sharding_propagation",
+        ...                           device_num=2)
+        >>> def test_shard(x, y):
+        ...     return x + y
+        >>> x = Tensor(np.ones(shape=(32, 10)), dtype=ms.float32)
+        >>> y = Tensor(np.ones(shape=(32, 10)), dtype=ms.float32)
+        >>> output = ms.shard(test_shard, in_strategy=((2, 1), (2, 1)))(x, y)
+        >>> print(output.shape)
+        (32, 10)
+
+    Tutorial Examples:
+        - `Functional Operator Sharding
+          <https://www.mindspore.cn/tutorials/experts/en/master/parallel/shard_function_parallel.html>`_
+    """
+    if not isinstance(fn, (ms.nn.Cell)):
+        logger.warning("'fn' is not a mindspore.nn.Cell, and its definition cannot involve Parameter; "
+                       "otherwise, the result may be incorrect.")
+    return Shard()(fn, in_strategy, out_strategy, parameter_plan, device, level)

mindspore/profiler/__init__.py

@@ -0,0 +1,27 @@
+# Copyright 2020-2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+This module provides Python APIs to enable the profiling of MindSpore neural networks.
+Users can import the mindspore.profiler.Profiler, initialize the Profiler object to start profiling,
+and use Profiler.analyse() to stop profiling and analyse the results.
+Users can visualize the results using the MindInsight tool.
+Now, Profiler supports AICORE operator, AICPU operator, HostCPU operator, memory,
+correspondence, cluster, etc data analysis.
+"""
+__all__ = ["Profiler", "EnvProfiler", "ProfilerLevel"]
+
+from mindspore.profiler.profiling import Profiler
+from mindspore.profiler.envprofiling import EnvProfiler
+from mindspore.profiler.profiling import ProfilerLevel

mindspore/profiler/common/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2020 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================