mindspore 2.4.10__cp39-cp39-win_amd64.whl → 2.6.0__cp39-cp39-win_amd64.whl

mindspore/ops_generate/resources/resource_list.py

@@ -0,0 +1,30 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Resource list."""
+
+from enum import Enum
+
+
+class ResourceType(Enum):
+    OP_PROTO = 0
+    DEPRECATED_OP_PROTO = 1
+    OP_DOC_YAML = 2
+    TENSOR_METHOD_DOC_YAML = 3
+    MINT_FUNC_DOC_YAML = 4
+    TENSOR_METHOD_PROTOS = 5
+    MINT_FUNC_PROTOS = 6
+    ALIAS_API_MAPPING = 7
+    FUNC_OP_PROTO = 8

mindspore/ops_generate/resources/resource_loader.py

@@ -0,0 +1,36 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Module of base class for resource loader."""
+
+from abc import ABC, abstractmethod
+from typing import Dict
+
+from .resource_list import ResourceType
+
+
+class ResourceLoader(ABC):
+    """
+    Abstract class for resource loader.
+    """
+    @abstractmethod
+    def load(self) -> Dict[ResourceType, object]:
+        """
+        Load resource.
+
+        Returns:
+            Dict[ResourceType, object]: The resource type and resource object map.
+        """
+        raise NotImplementedError

mindspore/ops_generate/resources/resource_manager.py

@@ -0,0 +1,64 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Module managing resource."""
+
+from common.op_proto import OpProtoLoader, DeprecatedOpProtoLoader, FuncOpProtoLoader
+from api.op_api_proto import OpApiProtoLoader
+
+from .resource_loader import ResourceLoader
+from .resource_list import ResourceType
+from .yaml_loader import OpDocYamlLoader, TensorMethodDocYamlLoader, MintFuncDocYamlLoader
+
+
+class ResourceManager():
+    """
+    ResourceManager is a class for managing resources.
+    """
+
+    def __init__(self):
+        self.resource_map = {}
+
+    def register_resource(self, loader: ResourceLoader) -> None:
+        """
+        Register resource.
+        """
+        self.resource_map.update(loader.load())
+
+    def get_resource(self, type: ResourceType) -> object:
+        """
+        Get resource by type.
+        """
+        if type not in self.resource_map:
+            raise ValueError(f"Resource '{type.name}' not registered")
+        return self.resource_map[type]
+
+
+def prepare_resources() -> ResourceManager:
+    """
+    Load needed resources.
+    """
+    resource_mgr = ResourceManager()
+    resource_mgr.register_resource(OpProtoLoader())
+    resource_mgr.register_resource(DeprecatedOpProtoLoader())
+    resource_mgr.register_resource(FuncOpProtoLoader())
+    resource_mgr.register_resource(OpDocYamlLoader())
+    resource_mgr.register_resource(TensorMethodDocYamlLoader())
+    resource_mgr.register_resource(MintFuncDocYamlLoader())
+    resource_mgr.register_resource(OpApiProtoLoader(
+        resource_mgr.get_resource(ResourceType.OP_PROTO),
+        resource_mgr.get_resource(ResourceType.DEPRECATED_OP_PROTO),
+        resource_mgr.get_resource(ResourceType.FUNC_OP_PROTO)))
+    return resource_mgr

mindspore/ops_generate/resources/yaml_loader.py

@@ -0,0 +1,88 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+
+"""Module loading ops yaml."""
+
+import os
+from typing import Sequence, Union
+
+from common.gen_utils import safe_load_yaml_from_dir
+import common.gen_constants as K
+
+from .resource_loader import ResourceLoader
+from .resource_list import ResourceType
+
+
+class YamlLoader(ResourceLoader):
+    """
+    YamlLoader is a utility class for loading yaml files.
+    """
+    def __init__(self, resouce_type: ResourceType, yaml_path: Union[Sequence[str], str]):
+        """
+        Initialize YamlLoader.
+
+        Args:
+            resouce_type (ResourceType): The type of the resource.
+            yaml_path (Union[Sequence[str], str]): The path to the yaml file or directory.
+        """
+        self.type = resouce_type
+        if isinstance(yaml_path, str):
+            self.yaml_path = [yaml_path]
+        else:
+            self.yaml_path = yaml_path
+
+    def load(self) -> dict:
+        """
+        Load yaml files.
+
+        Returns:
+            tuple[int, object]: The resource id and the yaml dict.
+        """
+        for yaml_path in self.yaml_path:
+            if not os.path.isdir(yaml_path):
+                raise ValueError(f"yaml path '{yaml_path}' not found")
+
+        yaml_dict = {}
+        for yaml_path in self.yaml_path:
+            yaml_dict.update(safe_load_yaml_from_dir(yaml_path))
+
+        return {self.type: yaml_dict}
+
+
+class OpDocYamlLoader(YamlLoader):
+    """
+    OpDocYamlLoader is a class for loading op primitive doc yaml files.
+    """
+    def __init__(self):
+        op_doc_yaml_path = os.path.join(K.WORK_DIR, K.MS_OP_DEF_YAML_PATH, "doc")
+        super().__init__(ResourceType.OP_DOC_YAML, op_doc_yaml_path)
+
+
+class TensorMethodDocYamlLoader(YamlLoader):
+    """
+    TensorMethodDocYamlLoader is a class for loading tensor method doc yaml files.
+    """
+    def __init__(self):
+        tensor_method_doc_yaml_path = os.path.join(K.WORK_DIR, K.MS_TENSOR_METHOD_DOC_YAML_PATH)
+        super().__init__(ResourceType.TENSOR_METHOD_DOC_YAML, tensor_method_doc_yaml_path)
+
+
+class MintFuncDocYamlLoader(YamlLoader):
+    """
+    MintFuncDocYamlLoader is a class for loading mint func doc yaml files.
+    """
+    def __init__(self):
+        mint_func_doc_yaml_path = os.path.join(K.WORK_DIR, K.MS_MINT_FUNC_DOC_YAML_PATH)
+        super().__init__(ResourceType.MINT_FUNC_DOC_YAML, mint_func_doc_yaml_path)

mindspore/ops_generate/tensor_py_cc_generator.py

@@ -0,0 +1,122 @@
+# Copyright 2025 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""
+Generates mindspore/ccsrc/pybind_api/ir/tensor_py.cc which includes the CPython Tensor APIs.
+"""
+
+import os
+import common.gen_constants as K
+from common.gen_utils import save_file
+import common.template as template
+from common.template import Template
+from common.base_generator import BaseGenerator
+from pyboost import pyboost_utils
+
+class TensorPyCppGenerator(BaseGenerator):
+    """
+    This class is responsible for generating mindspore/ccsrc/pybind_api/ir/tensor_register/
+    auto_generate/tensor_py_gen.cc
+    """
+    def __init__(self):
+        self.TENSOR_PY_CC_TEMPLATE = template.TENSOR_PY_CC_TEMPLATE
+        self.TENSOR_PY_H_TEMPLATE = template.TENSOR_PY_H_TEMPLATE
+        self.cpy_wrapper_template = Template("  DEFINE_TENSOR_METHOD_CPYWRAPPER(${pascal_api_name}) \\")
+        self.tensor_api_def_template = Template(
+            '{"${snake_api_name}"'
+            ', (PyCFunction)TensorMethod${pascal_api_name}_CPyWrapper, METH_VARARGS | METH_KEYWORDS},'
+        )
+        self.stubtensor_api_def_template = Template(
+            'py::cpp_function TensorMethod${snake_api_name}_wrapper(\n'
+            '  [](const py::object& self, const py::args& args, const py::kwargs& kwargs) {\n'
+            '      return  TensorMethod${pascal_api_name}(self, args, kwargs);\n'
+            '  },\n'
+            '  py::is_method(stubTensorClass)\n'
+            ');\n'
+            'stubTensorClass.attr("${snake_api_name}") = TensorMethod${snake_api_name}_wrapper;'
+        )
+
+    def generate(self, work_path, tensor_method_protos, alias_func_mapping):
+        """
+        Generates the content for the helper file and saves it to the specified path.
+
+        Args:
+            work_path (str): The directory where the generated file will be saved.
+            tensor_method_protos (dict): A dict mapping from Tensor func API names to their proto lists.
+            alias_func_mapping (dict): A dictionary mapping function name to its alias function names.
+
+        Returns:
+            None
+        """
+        wrapper_defs = []
+        tensor_api_defs = []
+        stubtensor_api_defs = []
+        for api_name, _ in tensor_method_protos.items():
+            pascal_api_name = pyboost_utils.format_func_api_name(api_name)
+            snake_api_name = api_name
+            wrapper_defs.append(self.cpy_wrapper_template.replace(pascal_api_name=pascal_api_name))
+            tensor_api_defs.append(
+                self.tensor_api_def_template.replace(
+                    snake_api_name=snake_api_name,
+                    pascal_api_name=pascal_api_name
+                )
+            )
+            stubtensor_api_defs.append(
+                self.stubtensor_api_def_template.replace(
+                    snake_api_name=snake_api_name,
+                    pascal_api_name=pascal_api_name
+                )
+            )
+            if api_name in alias_func_mapping:
+                alias_api_names = alias_func_mapping[api_name]
+                for alias_api_name in alias_api_names:
+                    snake_api_name = alias_api_name
+                    tensor_api_defs.append(
+                        self.tensor_api_def_template.replace(
+                            snake_api_name=snake_api_name,
+                            pascal_api_name=pascal_api_name
+                        )
+                    )
+                    stubtensor_api_defs.append(
+                        self.stubtensor_api_def_template.replace(
+                            snake_api_name=snake_api_name,
+                            pascal_api_name=pascal_api_name
+                        )
+                    )
+
+        # delete the ' \' for the last wrapper macro definition
+        wrapper_defs[-1] = wrapper_defs[-1][:-2]
+
+        file_str = self.TENSOR_PY_CC_TEMPLATE.replace(
+            tensor_api_defs=tensor_api_defs,
+            stubtensor_api_defs=stubtensor_api_defs
+        )
+        save_file(
+            os.path.join(work_path, K.TENSOR_PY_CC_PATH),
+            "tensor_py_gen.cc",
+            file_str
+        )
+
+        file_str = self.TENSOR_PY_H_TEMPLATE.replace(CPyWrapper_defs=wrapper_defs)
+        save_file(
+            os.path.join(work_path, K.TENSOR_PY_CC_PATH),
+            "tensor_py_gen.h",
+            file_str
+        )
+
+def _format_api_name(api_name):
+    has_suffix = api_name.endswith("_")
+    parts = api_name.strip("_").split("_")
+    formatted_api_name = "".join(part.capitalize() for part in parts)
+    return formatted_api_name + '_' if has_suffix else formatted_api_name

mindspore/parallel/__init__.py

@@ -19,12 +19,16 @@ from mindspore.parallel.algo_parameter_config import get_algo_parameters, reset_
     set_algo_parameters
 from mindspore.parallel.checkpoint_transform import rank_list_for_transform, transform_checkpoint_by_rank, \
     transform_checkpoints, merge_pipeline_strategys, sync_pipeline_shared_parameters, \
-    load_segmented_checkpoints
+    load_segmented_checkpoints, set_op_strategy_config, load_distributed_checkpoint, \
+    merge_sliced_parameter, restore_group_info_list, build_searched_strategy
+from mindspore.parallel.checkpoint_convert import rank_list_for_convert, convert_checkpoint_by_rank, convert_checkpoints
 from mindspore.parallel.parameter_broadcast import parameter_broadcast
 from mindspore.parallel.shard import shard, Layout
-from mindspore.parallel.transform_safetensors import safetensors_to_ckpt, ckpt_to_safetensors, unified_safetensors
+from mindspore.parallel.transform_safetensors import unified_safetensors
 
 __all__ = ["set_algo_parameters", "reset_algo_parameters", "get_algo_parameters", "rank_list_for_transform",
            "transform_checkpoint_by_rank", "transform_checkpoints", "merge_pipeline_strategys", "shard",
            "sync_pipeline_shared_parameters", "Layout", "parameter_broadcast", "load_segmented_checkpoints",
-           "safetensors_to_ckpt", "ckpt_to_safetensors", "unified_safetensors"]
+           "unified_safetensors", "load_distributed_checkpoint", "merge_sliced_parameter", "restore_group_info_list",
+           "build_searched_strategy", "set_op_strategy_config", "rank_list_for_convert",
+           "convert_checkpoint_by_rank", "convert_checkpoints"]