mindspore 2.2.0__cp37-cp37m-manylinux1_x86_64.whl → 2.2.11__cp37-cp37m-manylinux1_x86_64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mindspore/.commit_id +1 -1
- mindspore/_akg/akg/composite/build_module.py +104 -20
- mindspore/_akg/akg/utils/ascend_profilier/cann_file_parser.py +76 -0
- mindspore/_akg/akg/utils/ascend_profilier/file_manager.py +56 -0
- mindspore/_akg/akg/utils/ascend_profilier/op_summary_bean.py +23 -0
- mindspore/_akg/akg/utils/ascend_profilier/op_summary_headers.py +8 -0
- mindspore/_akg/akg/utils/ascend_profilier/op_summary_parser.py +42 -0
- mindspore/_akg/akg/utils/ascend_profilier/path_manager.py +65 -0
- mindspore/_akg/akg/utils/composite_op_helper.py +7 -2
- mindspore/_akg/akg/utils/dump_ascend_meta.py +22 -3
- mindspore/_akg/akg/utils/kernel_exec.py +41 -15
- mindspore/_akg/akg/utils/tbe_codegen_utils.py +27 -6
- mindspore/_akg/akg/utils/util.py +56 -1
- mindspore/_c_dataengine.cpython-37m-x86_64-linux-gnu.so +0 -0
- mindspore/_c_expression.cpython-37m-x86_64-linux-gnu.so +0 -0
- mindspore/_checkparam.py +3 -3
- mindspore/_extends/graph_kernel/model/graph_split.py +84 -76
- mindspore/_extends/graph_kernel/splitter.py +3 -2
- mindspore/_extends/parallel_compile/akg_compiler/build_tbe_kernel.py +83 -66
- mindspore/_extends/parallel_compile/akg_compiler/tbe_topi.py +4 -4
- mindspore/_extends/parallel_compile/akg_compiler/util.py +10 -7
- mindspore/_extends/parallel_compile/tbe_compiler/tbe_helper.py +2 -1
- mindspore/_extends/parse/__init__.py +3 -2
- mindspore/_extends/parse/parser.py +6 -1
- mindspore/_extends/parse/standard_method.py +14 -11
- mindspore/_extends/remote/kernel_build_server.py +2 -1
- mindspore/_mindspore_offline_debug.cpython-37m-x86_64-linux-gnu.so +0 -0
- mindspore/bin/cache_admin +0 -0
- mindspore/bin/cache_server +0 -0
- mindspore/common/_utils.py +16 -0
- mindspore/common/api.py +1 -1
- mindspore/common/auto_dynamic_shape.py +81 -85
- mindspore/common/dump.py +1 -1
- mindspore/common/tensor.py +3 -20
- mindspore/config/op_info.config +1 -1
- mindspore/context.py +11 -4
- mindspore/dataset/engine/cache_client.py +8 -5
- mindspore/dataset/engine/datasets_standard_format.py +5 -0
- mindspore/dataset/vision/transforms.py +21 -21
- mindspore/experimental/optim/adam.py +1 -1
- mindspore/gen_ops.py +1 -1
- mindspore/include/api/model.h +17 -0
- mindspore/include/api/status.h +8 -3
- mindspore/lib/libdnnl.so.2 +0 -0
- mindspore/lib/libmindspore.so +0 -0
- mindspore/lib/libmindspore_backend.so +0 -0
- mindspore/lib/libmindspore_common.so +0 -0
- mindspore/lib/libmindspore_core.so +0 -0
- mindspore/lib/libmindspore_glog.so.0 +0 -0
- mindspore/lib/libmindspore_gpr.so.15 +0 -0
- mindspore/lib/libmindspore_grpc++.so.1 +0 -0
- mindspore/lib/libmindspore_grpc.so.15 +0 -0
- mindspore/lib/libmindspore_shared_lib.so +0 -0
- mindspore/lib/libnnacl.so +0 -0
- mindspore/lib/libopencv_core.so.4.5 +0 -0
- mindspore/lib/libopencv_imgcodecs.so.4.5 +0 -0
- mindspore/lib/libopencv_imgproc.so.4.5 +0 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend310/aic-ascend310-ops-info.json +123 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend310p/aic-ascend310p-ops-info.json +123 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend910/aic-ascend910-ops-info.json +158 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend910b/aic-ascend910b-ops-info.json +37 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/add_dsl.py +46 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/add_tik.py +51 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/kv_cache_mgr.py +241 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/matmul_tik.py +212 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/vector_core/tbe/custom_aicore_ops_impl/add_dsl.py +46 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/vector_core/tbe/custom_aicore_ops_impl/add_tik.py +51 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/vector_core/tbe/custom_aicore_ops_impl/kv_cache_mgr.py +241 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/vector_core/tbe/custom_aicore_ops_impl/matmul_tik.py +212 -0
- mindspore/lib/plugin/ascend/custom_aicore_ops/op_proto/libop_proto.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_aicpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_cpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/config/cust_aicpu_kernel.json +78 -80
- mindspore/lib/plugin/ascend/custom_aicpu_ops/op_proto/libcust_op_proto.so +0 -0
- mindspore/lib/plugin/ascend/libakg.so +0 -0
- mindspore/lib/plugin/ascend/libhccl_plugin.so +0 -0
- mindspore/lib/plugin/ascend/libmindspore_aicpu_kernels.so +0 -0
- mindspore/lib/plugin/ascend/libmindspore_cpu_kernels.so +0 -0
- mindspore/lib/plugin/cpu/libakg.so +0 -0
- mindspore/lib/plugin/gpu/libcuda_ops.so.10 +0 -0
- mindspore/lib/plugin/gpu/libcuda_ops.so.11 +0 -0
- mindspore/lib/plugin/gpu10.1/libakg.so +0 -0
- mindspore/lib/plugin/gpu10.1/libnccl.so.2 +0 -0
- mindspore/lib/plugin/gpu11.1/libakg.so +0 -0
- mindspore/lib/plugin/gpu11.6/libakg.so +0 -0
- mindspore/lib/plugin/gpu11.6/libnccl.so.2 +0 -0
- mindspore/lib/plugin/libmindspore_ascend.so.1 +0 -0
- mindspore/lib/plugin/libmindspore_ascend.so.2 +0 -0
- mindspore/lib/plugin/libmindspore_gpu.so.10.1 +0 -0
- mindspore/lib/plugin/libmindspore_gpu.so.11.1 +0 -0
- mindspore/lib/plugin/libmindspore_gpu.so.11.6 +0 -0
- mindspore/nn/cell.py +0 -3
- mindspore/nn/layer/activation.py +4 -5
- mindspore/nn/layer/conv.py +39 -23
- mindspore/nn/layer/flash_attention.py +54 -129
- mindspore/nn/layer/math.py +3 -7
- mindspore/nn/layer/rnn_cells.py +5 -5
- mindspore/nn/wrap/__init__.py +4 -2
- mindspore/nn/wrap/cell_wrapper.py +12 -3
- mindspore/numpy/utils_const.py +5 -5
- mindspore/ops/_grad_experimental/grad_array_ops.py +1 -1
- mindspore/ops/_grad_experimental/grad_implementations.py +2 -2
- mindspore/ops/_grad_experimental/grad_math_ops.py +19 -18
- mindspore/ops/_grad_experimental/grad_sparse_ops.py +3 -3
- mindspore/ops/_op_impl/aicpu/add.py +3 -3
- mindspore/ops/_op_impl/aicpu/linear_sum_assignment.py +21 -2
- mindspore/ops/_utils/utils.py +2 -0
- mindspore/ops/composite/multitype_ops/_compile_utils.py +2 -1
- mindspore/ops/composite/multitype_ops/getitem_impl.py +2 -2
- mindspore/ops/function/array_func.py +10 -7
- mindspore/ops/function/grad/grad_func.py +0 -1
- mindspore/ops/function/nn_func.py +98 -9
- mindspore/ops/function/random_func.py +2 -1
- mindspore/ops/op_info_register.py +24 -21
- mindspore/ops/operations/__init__.py +6 -2
- mindspore/ops/operations/_grad_ops.py +25 -6
- mindspore/ops/operations/_inner_ops.py +155 -23
- mindspore/ops/operations/array_ops.py +9 -7
- mindspore/ops/operations/comm_ops.py +2 -2
- mindspore/ops/operations/custom_ops.py +85 -68
- mindspore/ops/operations/inner_ops.py +26 -3
- mindspore/ops/operations/math_ops.py +7 -6
- mindspore/ops/operations/nn_ops.py +193 -49
- mindspore/parallel/_parallel_serialization.py +10 -3
- mindspore/parallel/_tensor.py +4 -1
- mindspore/parallel/checkpoint_transform.py +13 -2
- mindspore/parallel/shard.py +17 -10
- mindspore/profiler/common/util.py +1 -0
- mindspore/profiler/parser/ascend_hccl_generator.py +232 -0
- mindspore/profiler/parser/ascend_msprof_exporter.py +86 -43
- mindspore/profiler/parser/ascend_msprof_generator.py +196 -9
- mindspore/profiler/parser/ascend_op_generator.py +1 -1
- mindspore/profiler/parser/ascend_timeline_generator.py +6 -182
- mindspore/profiler/parser/base_timeline_generator.py +1 -1
- mindspore/profiler/parser/cpu_gpu_timeline_generator.py +2 -2
- mindspore/profiler/parser/framework_parser.py +1 -1
- mindspore/profiler/parser/profiler_info.py +19 -0
- mindspore/profiler/profiling.py +46 -24
- mindspore/rewrite/api/pattern_engine.py +1 -1
- mindspore/rewrite/parsers/for_parser.py +7 -7
- mindspore/rewrite/parsers/module_parser.py +4 -4
- mindspore/rewrite/symbol_tree.py +1 -4
- mindspore/run_check/_check_version.py +5 -3
- mindspore/safeguard/rewrite_obfuscation.py +52 -28
- mindspore/scipy/ops.py +55 -5
- mindspore/scipy/optimize/__init__.py +3 -2
- mindspore/scipy/optimize/linear_sum_assignment.py +38 -33
- mindspore/train/callback/_summary_collector.py +1 -1
- mindspore/train/dataset_helper.py +1 -0
- mindspore/train/model.py +2 -2
- mindspore/train/serialization.py +97 -11
- mindspore/train/summary/_summary_adapter.py +1 -1
- mindspore/train/summary/summary_record.py +23 -7
- mindspore/version.py +1 -1
- {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/METADATA +3 -2
- {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/RECORD +160 -151
- mindspore/ops/_op_impl/_custom_op/flash_attention/attention.py +0 -406
- mindspore/ops/_op_impl/_custom_op/flash_attention/constants.py +0 -41
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_bwd.py +0 -467
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_fwd.py +0 -563
- mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_impl.py +0 -193
- mindspore/ops/_op_impl/_custom_op/flash_attention/tik_ops_utils.py +0 -435
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/__init__.py +0 -0
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/sparse_tiling.py +0 -45
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/strategy.py +0 -67
- mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/wukong_tiling.py +0 -62
- /mindspore/{ops/_op_impl/_custom_op/flash_attention → _akg/akg/utils/ascend_profilier}/__init__.py +0 -0
- {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/WHEEL +0 -0
- {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/entry_points.txt +0 -0
- {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2021 Huawei Technologies Co., Ltd
|
|
1
|
+
# Copyright 2021-2023 Huawei Technologies Co., Ltd
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -15,5 +15,6 @@
|
|
|
15
15
|
"""Optimize submodule"""
|
|
16
16
|
from .minimize import minimize
|
|
17
17
|
from .line_search import line_search
|
|
18
|
+
from .linear_sum_assignment import linear_sum_assignment
|
|
18
19
|
|
|
19
|
-
__all__ = ["minimize", "line_search"]
|
|
20
|
+
__all__ = ["minimize", "line_search", "linear_sum_assignment"]
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright
|
|
1
|
+
# Copyright 2023 Huawei Technologies Co., Ltd
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -15,59 +15,64 @@
|
|
|
15
15
|
"""Linear Sum Assignment"""
|
|
16
16
|
import sys
|
|
17
17
|
from ..ops import LinearSumAssignment
|
|
18
|
-
from ...common import dtype as mstype
|
|
19
|
-
from ..utils import _mstype_check, _dtype_check
|
|
20
18
|
from ... import Tensor
|
|
21
19
|
|
|
22
20
|
|
|
23
|
-
def
|
|
24
|
-
"""
|
|
21
|
+
def linear_sum_assignment(cost_matrix, maximize, dimension_limit=Tensor(sys.maxsize)):
|
|
22
|
+
r"""
|
|
25
23
|
Solve the linear sum assignment problem.
|
|
26
24
|
|
|
25
|
+
The assignment problem is represented as follows:
|
|
26
|
+
|
|
27
|
+
.. math::
|
|
28
|
+
min\sum_{i}^{} \sum_{j}^{} C_{i,j} X_{i,j}
|
|
29
|
+
|
|
30
|
+
where :math:`C` is cost matrix, :math:`X_{i,j} = 1` means column :math:`j` is assigned to row :math:`i` .
|
|
31
|
+
|
|
27
32
|
Args:
|
|
28
|
-
cost_matrix: 2-D
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
A scalar used to limit the actual size of the 1st dimension. Optimized for
|
|
34
|
-
padding scenes. Default means no dimension limit.
|
|
33
|
+
cost_matrix (Tensor): 2-D cost matrix. Tensor of shape :math:`(M, N)` .
|
|
34
|
+
maximize (bool): Calculate a maximum weight matching if true, otherwise calculate a minimum weight matching.
|
|
35
|
+
dimension_limit (Tensor, optional): A scalar used to limit the actual size of the 2nd dimension of
|
|
36
|
+
``cost_matrix``. Default is ``Tensor(sys.maxsize)``, which means no limitation. The type is 0-D int64
|
|
37
|
+
Tensor.
|
|
35
38
|
|
|
36
39
|
Returns:
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
+
A tuple of tensors containing 'row_idx' and 'col_idx'.
|
|
41
|
+
|
|
42
|
+
- **row_idx** (Tensor) - Row indices of the problem. If `dimension_limit` is given, -1 would be padded at the
|
|
43
|
+
end. The shape is :math:`(N, )` , where :math:`N` is the minimum value of `cost_matrix` dimension.
|
|
44
|
+
- **col_idx** (Tensor) - Column indices of the problem. If `dimension_limit` is given, -1 would be padded at
|
|
45
|
+
the end. The shape is :math:`(N, )` , where :math:`N` is the minimum value of `cost_matrix` dimension.
|
|
46
|
+
|
|
47
|
+
Raises:
|
|
48
|
+
TypeError: If the data type of `cost_matrix` is not the type in [float16, float32, float64,
|
|
49
|
+
int8, int16, int32, int64, uint8, uint16, uint32, uint64, bool]
|
|
50
|
+
TypeError: If the type of `maximize` is not bool.
|
|
51
|
+
TypeError: If the data type of `dimension_limit` is not int64.
|
|
52
|
+
ValueError: If the rank of `cost_matrix` is not 2.
|
|
53
|
+
|
|
40
54
|
|
|
41
55
|
Supported Platforms:
|
|
42
|
-
``CPU``
|
|
56
|
+
``Ascend`` ``CPU``
|
|
43
57
|
|
|
44
|
-
|
|
58
|
+
Examples:
|
|
45
59
|
>>> import mindspore as ms
|
|
46
60
|
>>> import numpy as np
|
|
47
61
|
>>> from mindspore import Tensor
|
|
48
|
-
>>>
|
|
49
|
-
>>> cost_matrix = Tensor(np.array([[2, 3, 3], [3, 2, 3], [3, 3, 2]])).astype(
|
|
62
|
+
>>> import mindspore.scipy.optimize.linear_sum_assignment as lsap
|
|
63
|
+
>>> cost_matrix = Tensor(np.array([[2, 3, 3], [3, 2, 3], [3, 3, 2]])).astype(ms.float64)
|
|
50
64
|
>>> dimension_limit = Tensor(2)
|
|
51
|
-
>>> maximize =
|
|
65
|
+
>>> maximize = False
|
|
52
66
|
>>> a, b = lsap(cost_matrix, maximize, dimension_limit)
|
|
53
67
|
>>> print(a)
|
|
54
|
-
[
|
|
68
|
+
[0 1 -1]
|
|
55
69
|
>>> print(b)
|
|
56
|
-
[
|
|
70
|
+
[0 1 -1]
|
|
57
71
|
>>> a, b = lsap(cost_matrix, maximize)
|
|
58
72
|
>>> print(a)
|
|
59
|
-
[
|
|
73
|
+
[0 1 2]
|
|
60
74
|
>>> print(b)
|
|
61
|
-
[
|
|
75
|
+
[0 1 2]
|
|
62
76
|
"""
|
|
63
|
-
func_name = 'linear_sum_assignment'
|
|
64
|
-
_mstype_check(func_name, cost_matrix, mstype.TensorType, 'cost_matrix')
|
|
65
|
-
_mstype_check(func_name, dimension_limit,
|
|
66
|
-
mstype.TensorType, 'dimension_limit')
|
|
67
|
-
_mstype_check(func_name, maximize, mstype.TensorType, 'maximize')
|
|
68
|
-
_dtype_check(func_name, cost_matrix, [mstype.float32, mstype.float64])
|
|
69
|
-
_dtype_check(func_name, dimension_limit, [mstype.int64])
|
|
70
|
-
_dtype_check(func_name, maximize, [mstype.bool_])
|
|
71
|
-
|
|
72
77
|
solve = LinearSumAssignment()
|
|
73
78
|
return solve(cost_matrix, dimension_limit, maximize)
|
|
@@ -1157,7 +1157,7 @@ class SummaryCollector(Callback):
|
|
|
1157
1157
|
except TypeError as exc:
|
|
1158
1158
|
logger.warning("Summary cannot collect the type of metrics, currently support type: dict, list, tuple, "
|
|
1159
1159
|
"str, int, float, bool and None. %s.", str(exc))
|
|
1160
|
-
self._parse_dataset(cb_params, eval_lineage)
|
|
1160
|
+
_ = self._parse_dataset(cb_params, eval_lineage)
|
|
1161
1161
|
|
|
1162
1162
|
eval_lineage_message = self._package_eval_lineage_message(eval_lineage)
|
|
1163
1163
|
self._record.add_value(PluginEnum.EVAL_LINEAGE.value, 'eval_lineage', eval_lineage_message)
|
|
@@ -52,6 +52,7 @@ def _dynamic_sink_data(dataset, dataset_iter):
|
|
|
52
52
|
if hasattr(dataset_iter, "sink_size") and \
|
|
53
53
|
dataset_iter.sink_size == 1 and \
|
|
54
54
|
dataset.get_dataset_size() != 1 and \
|
|
55
|
+
not hasattr(dataset, "__no_send__") and \
|
|
55
56
|
hasattr(dataset_iter, "sink_count") and \
|
|
56
57
|
dataset_iter.sink_count == 1:
|
|
57
58
|
return True
|
mindspore/train/model.py
CHANGED
|
@@ -1586,7 +1586,7 @@ class Model:
|
|
|
1586
1586
|
use_past = not is_first_iteration
|
|
1587
1587
|
model_group_id = self._mindspore_lite_model_group_id
|
|
1588
1588
|
|
|
1589
|
-
check_input_data(*predict_data, data_class=Tensor)
|
|
1589
|
+
check_input_data(*predict_data, data_class=(int, float, str, None, Tensor))
|
|
1590
1590
|
if use_past:
|
|
1591
1591
|
# Execute incremental model inference
|
|
1592
1592
|
if not self._lite_incremental_predictor:
|
|
@@ -1867,7 +1867,7 @@ class Model:
|
|
|
1867
1867
|
if _get_parallel_mode() not in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL):
|
|
1868
1868
|
raise RuntimeError('Infer predict layout only supports semi auto parallel and auto parallel mode.')
|
|
1869
1869
|
_parallel_predict_check()
|
|
1870
|
-
check_input_data(*predict_data, data_class=Tensor)
|
|
1870
|
+
check_input_data(*predict_data, data_class=(int, float, str, None, Tensor))
|
|
1871
1871
|
|
|
1872
1872
|
predict_net = self._predict_network
|
|
1873
1873
|
# Unlike the cases in build_train_network() and build_eval_network(), 'multi_subgraphs' is not set
|
mindspore/train/serialization.py
CHANGED
|
@@ -65,15 +65,17 @@ from mindspore.train._utils import read_proto
|
|
|
65
65
|
from mindspore._c_expression import load_mindir, _encrypt, _decrypt, _is_cipher_file, dynamic_obfuscate_mindir, \
|
|
66
66
|
split_mindir, split_dynamic_mindir
|
|
67
67
|
from ..ops.operations._opaque_predicate_registry import add_opaque_predicate, clean_funcs
|
|
68
|
+
from ..ops.operations import Cast
|
|
68
69
|
|
|
69
70
|
tensor_to_ms_type = {"Int8": mstype.int8, "UInt8": mstype.uint8, "Int16": mstype.int16, "UInt16": mstype.uint16,
|
|
70
71
|
"Int32": mstype.int32, "UInt32": mstype.uint32, "Int64": mstype.int64, "UInt64": mstype.uint64,
|
|
71
72
|
"Float16": mstype.float16, "Float32": mstype.float32, "Float64": mstype.float64,
|
|
72
|
-
"Bool": mstype.bool_, "str": mstype.string}
|
|
73
|
+
"Bool": mstype.bool_, "str": mstype.string, "BFloat16": mstype.bfloat16}
|
|
73
74
|
|
|
74
75
|
tensor_to_np_type = {"Int8": np.int8, "UInt8": np.uint8, "Int16": np.int16, "UInt16": np.uint16,
|
|
75
76
|
"Int32": np.int32, "UInt32": np.uint32, "Int64": np.int64, "UInt64": np.uint64,
|
|
76
|
-
"Float16": np.float16, "Float32": np.float32, "Float64": np.float64, "Bool": np.bool_, "str": "U"
|
|
77
|
+
"Float16": np.float16, "Float32": np.float32, "Float64": np.float64, "Bool": np.bool_, "str": "U",
|
|
78
|
+
"BFloat16": np.float32}
|
|
77
79
|
|
|
78
80
|
np_type_convert = {"int32": np.int32, "float32": np.float32, "float16": np.float16, "float64": np.float64}
|
|
79
81
|
|
|
@@ -91,6 +93,8 @@ PARAMETER_SPLIT_SIZE = 1024 * 1024 * 1024
|
|
|
91
93
|
ENCRYPT_BLOCK_SIZE = 64 * 1024
|
|
92
94
|
INT_64_MAX = 9223372036854775807
|
|
93
95
|
|
|
96
|
+
cpu_cast = Cast().set_device("CPU")
|
|
97
|
+
|
|
94
98
|
|
|
95
99
|
def _special_process_par(par, new_par):
|
|
96
100
|
"""
|
|
@@ -107,7 +111,11 @@ def _special_process_par(par, new_par):
|
|
|
107
111
|
if new_par.data.shape[par_shape_len + i] != 1:
|
|
108
112
|
return False
|
|
109
113
|
|
|
110
|
-
|
|
114
|
+
if new_par.data.dtype == mstype.bfloat16:
|
|
115
|
+
new_val = cpu_cast(new_par.data, mstype.float32).asnumpy()
|
|
116
|
+
else:
|
|
117
|
+
new_val = new_par.data.asnumpy()
|
|
118
|
+
|
|
111
119
|
new_val = new_val.reshape(par.data.shape)
|
|
112
120
|
par.set_data(Tensor(new_val, par.data.dtype))
|
|
113
121
|
return True
|
|
@@ -128,7 +136,10 @@ def _update_param(param, new_param, strict_load):
|
|
|
128
136
|
|
|
129
137
|
if param.data.dtype != new_param.data.dtype:
|
|
130
138
|
if _type_convert(param, new_param, strict_load):
|
|
131
|
-
|
|
139
|
+
if new_param.data.dtype == mstype.bfloat16:
|
|
140
|
+
new_tensor = cpu_cast(new_param.data, param.data.dtype)
|
|
141
|
+
else:
|
|
142
|
+
new_tensor = Tensor(new_param.data.asnumpy(), param.data.dtype)
|
|
132
143
|
param.set_data(new_tensor, param.sliced)
|
|
133
144
|
return
|
|
134
145
|
|
|
@@ -231,10 +242,16 @@ def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_
|
|
|
231
242
|
continue
|
|
232
243
|
if value[0] == "offload_parameter":
|
|
233
244
|
new_value = value[1:]
|
|
234
|
-
|
|
245
|
+
if value[3].dtype == mstype.bfloat16:
|
|
246
|
+
new_value[2] = cpu_cast(value[3], mstype.float32).asnumpy().reshape(-1)
|
|
247
|
+
else:
|
|
248
|
+
new_value[2] = value[3].asnumpy().reshape(-1)
|
|
235
249
|
_write_parameter_data(name, new_value, f, enc_key, plain_data)
|
|
236
250
|
_offload_if_config(value[3])
|
|
237
251
|
continue
|
|
252
|
+
if value[0] == "BFloat16_tensor":
|
|
253
|
+
_write_bfloat16_data(name, value, f, enc_key, plain_data)
|
|
254
|
+
continue
|
|
238
255
|
if isinstance(value[2], Tensor):
|
|
239
256
|
_write_hugeparameter(name, value, f)
|
|
240
257
|
continue
|
|
@@ -269,6 +286,21 @@ def _write_random_seed(name, value, f):
|
|
|
269
286
|
f.write(checkpoint_list.SerializeToString())
|
|
270
287
|
|
|
271
288
|
|
|
289
|
+
def _write_bfloat16_data(name, value, f, enc_key, plain_data):
|
|
290
|
+
"""Write bfloat16 data into protobuf file"""
|
|
291
|
+
checkpoint_list = Checkpoint()
|
|
292
|
+
param_value = checkpoint_list.value.add()
|
|
293
|
+
param_value.tag = name
|
|
294
|
+
param_tensor = param_value.tensor
|
|
295
|
+
param_tensor.dims.extend(value[1])
|
|
296
|
+
param_tensor.tensor_type = value[2]
|
|
297
|
+
param_tensor.tensor_content = value[3].get_bytes()
|
|
298
|
+
if enc_key is None:
|
|
299
|
+
f.write(checkpoint_list.SerializeToString())
|
|
300
|
+
else:
|
|
301
|
+
plain_data.write(checkpoint_list.SerializeToString())
|
|
302
|
+
|
|
303
|
+
|
|
272
304
|
def _write_parameter_data(name, value, f, enc_key, plain_data):
|
|
273
305
|
"""Write parameter data into protobuf file."""
|
|
274
306
|
data_size = value[2].nbytes / 1024
|
|
@@ -447,6 +479,10 @@ def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
|
|
|
447
479
|
elif param["data"][0] == "offload_parameter":
|
|
448
480
|
data_list[key].append("offload_parameter")
|
|
449
481
|
_save_param_list_data(data_list, key, param)
|
|
482
|
+
elif param["data"][0] == "BFloat16_tensor":
|
|
483
|
+
data_list[key].append("BFloat16_tensor")
|
|
484
|
+
_save_param_list_data(data_list, key, param)
|
|
485
|
+
continue
|
|
450
486
|
|
|
451
487
|
if isinstance(param["data"], str):
|
|
452
488
|
data_list[key].append([0])
|
|
@@ -456,6 +492,15 @@ def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
|
|
|
456
492
|
else:
|
|
457
493
|
if isinstance(param["data"], Parameter):
|
|
458
494
|
param["data"].init_data()
|
|
495
|
+
if isinstance(param["data"], Tensor) and param["data"].dtype == mstype.bfloat16:
|
|
496
|
+
data_list[key].append("BFloat16_tensor")
|
|
497
|
+
dims = []
|
|
498
|
+
for dim in param["data"].shape:
|
|
499
|
+
dims.append(dim)
|
|
500
|
+
data_list[key].append(dims)
|
|
501
|
+
data_list[key].append("BFloat16")
|
|
502
|
+
data_list[key].append(cpu_cast(param["data"], mstype.float32))
|
|
503
|
+
continue
|
|
459
504
|
dims = []
|
|
460
505
|
if param['data'].shape == ():
|
|
461
506
|
dims.append(0)
|
|
@@ -465,7 +510,10 @@ def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
|
|
|
465
510
|
data_list[key].append(dims)
|
|
466
511
|
tensor_type = str(param["data"].dtype)
|
|
467
512
|
data_list[key].append(tensor_type)
|
|
468
|
-
|
|
513
|
+
if param["data"].dtype == mstype.bfloat16:
|
|
514
|
+
data = cpu_cast(param["data"], mstype.float32).asnumpy().reshape(-1)
|
|
515
|
+
else:
|
|
516
|
+
data = param["data"].asnumpy().reshape(-1)
|
|
469
517
|
data_list[key].append(data)
|
|
470
518
|
|
|
471
519
|
if async_save:
|
|
@@ -571,6 +619,12 @@ def _convert_cell_to_param_list(save_obj, integrated_save, append_dict, choice_f
|
|
|
571
619
|
param_data.append(param_tensor.shape)
|
|
572
620
|
param_data.append(str(param_tensor.dtype))
|
|
573
621
|
param_data.append(value.key)
|
|
622
|
+
elif value.data.dtype == mstype.bfloat16:
|
|
623
|
+
param_data = ["BFloat16_tensor"]
|
|
624
|
+
param_data.append(cpu_cast(value.data, mstype.float32))
|
|
625
|
+
param_data.append(value.data.shape)
|
|
626
|
+
param_data.append("BFloat16")
|
|
627
|
+
param_data.append(value.key)
|
|
574
628
|
else:
|
|
575
629
|
param_data = Tensor(value.data.asnumpy())
|
|
576
630
|
|
|
@@ -1065,6 +1119,13 @@ def load_checkpoint(ckpt_file_name, net=None, strict_load=False, filter_prefix=N
|
|
|
1065
1119
|
if data_type == 'str':
|
|
1066
1120
|
str_length = int(len(data) / 4)
|
|
1067
1121
|
np_type = np_type + str(str_length)
|
|
1122
|
+
if data_type == "BFloat16":
|
|
1123
|
+
dims = element.tensor.dims
|
|
1124
|
+
param_data = np.frombuffer(data, np_type)
|
|
1125
|
+
param_data = param_data.reshape(list(dims))
|
|
1126
|
+
parameter = Parameter(Tensor(param_data, ms_type), name=element.tag)
|
|
1127
|
+
parameter_dict[element.tag] = parameter
|
|
1128
|
+
continue
|
|
1068
1129
|
element_data = np.frombuffer(data, np_type)
|
|
1069
1130
|
param_data_list.append(element_data)
|
|
1070
1131
|
if (element_id == len(checkpoint_list.value) - 1) or \
|
|
@@ -1806,8 +1867,15 @@ def _split_save(net_dict, model, file_name, is_encrypt, **kwargs):
|
|
|
1806
1867
|
data_file_name = os.path.join(dirname, external_local)
|
|
1807
1868
|
f, parameter_size, offset = _get_data_file(is_encrypt, kwargs, data_file_name)
|
|
1808
1869
|
try:
|
|
1870
|
+
round_ = 0
|
|
1871
|
+
names = []
|
|
1809
1872
|
for param_proto in model.graph.parameter:
|
|
1810
1873
|
name = param_proto.name[param_proto.name.find(":") + 1:]
|
|
1874
|
+
names.append((name, param_proto))
|
|
1875
|
+
names.sort(key=lambda x: x[0])
|
|
1876
|
+
for pairs in names:
|
|
1877
|
+
name = pairs[0]
|
|
1878
|
+
param_proto = pairs[1]
|
|
1811
1879
|
param = net_dict[name]
|
|
1812
1880
|
raw_data = param.data.get_bytes()
|
|
1813
1881
|
data_length = len(raw_data)
|
|
@@ -1827,6 +1895,8 @@ def _split_save(net_dict, model, file_name, is_encrypt, **kwargs):
|
|
|
1827
1895
|
offset += (data_length + append_size)
|
|
1828
1896
|
write_data = _encrypt_data(is_encrypt, write_data, kwargs)
|
|
1829
1897
|
f.write(write_data)
|
|
1898
|
+
round_ += 1
|
|
1899
|
+
logger.debug(f"writing {round_}th split data, name:{name}")
|
|
1830
1900
|
|
|
1831
1901
|
graph_file_name = os.path.join(dirname, file_prefix + "_graph.mindir")
|
|
1832
1902
|
if os.path.exists(graph_file_name):
|
|
@@ -2315,7 +2385,12 @@ def merge_sliced_parameter(sliced_parameters, strategy=None):
|
|
|
2315
2385
|
|
|
2316
2386
|
layerwise_parallel = sliced_parameters[0].layerwise_parallel
|
|
2317
2387
|
requires_grad = sliced_parameters[0].requires_grad
|
|
2318
|
-
sliced_data = [
|
|
2388
|
+
sliced_data = []
|
|
2389
|
+
for parameter in sliced_parameters:
|
|
2390
|
+
if parameter.data.dtype == mstype.bfloat16:
|
|
2391
|
+
sliced_data.append(cpu_cast(parameter.data, mstype.float32).asnumpy())
|
|
2392
|
+
else:
|
|
2393
|
+
sliced_data.append(parameter.data.asnumpy())
|
|
2319
2394
|
|
|
2320
2395
|
if not strategy:
|
|
2321
2396
|
merged_tensor = Tensor(np.concatenate(sliced_data))
|
|
@@ -2521,7 +2596,11 @@ def load_distributed_checkpoint(network, checkpoint_filenames, predict_strategy=
|
|
|
2521
2596
|
param_index = list(set(param_index))
|
|
2522
2597
|
param_index.sort()
|
|
2523
2598
|
for rank_num in param_index:
|
|
2524
|
-
|
|
2599
|
+
if param_total_dict[param.name][rank_num].data.dtype == mstype.bfloat16:
|
|
2600
|
+
param_stride.append(
|
|
2601
|
+
cpu_cast(param_total_dict[param.name][rank_num].data, mstype.float32).asnumpy())
|
|
2602
|
+
else:
|
|
2603
|
+
param_stride.append(param_total_dict[param.name][rank_num].data.asnumpy())
|
|
2525
2604
|
|
|
2526
2605
|
sliced_param = Parameter(Tensor(np.concatenate(param_stride)), name=param.name)
|
|
2527
2606
|
else:
|
|
@@ -2536,7 +2615,10 @@ def load_distributed_checkpoint(network, checkpoint_filenames, predict_strategy=
|
|
|
2536
2615
|
split_param = _merge_and_split(sliced_params, _param_unique_strategy, predict_strategy)
|
|
2537
2616
|
opt_shard_group = predict_strategy[param.name][5] if predict_strategy else None
|
|
2538
2617
|
if opt_shard_group:
|
|
2539
|
-
|
|
2618
|
+
if split_param.data.dtype == mstype.bfloat16:
|
|
2619
|
+
data = cpu_cast(split_param.data, mstype.float32).asnumpy()
|
|
2620
|
+
else:
|
|
2621
|
+
data = split_param.data.asnumpy()
|
|
2540
2622
|
rank = get_rank(opt_shard_group)
|
|
2541
2623
|
size = get_group_size(opt_shard_group)
|
|
2542
2624
|
try:
|
|
@@ -2638,7 +2720,11 @@ def _merge_and_split(sliced_params, train_strategy, predict_strategy):
|
|
|
2638
2720
|
split_tensor = _load_tensor(merged_param.data, tensor_layout[0], tensor_layout[1], rank)
|
|
2639
2721
|
requires_grad = merged_param.requires_grad
|
|
2640
2722
|
layerwise_parallel = merged_param.layerwise_parallel
|
|
2641
|
-
|
|
2723
|
+
data_type = merged_param.data.dtype
|
|
2724
|
+
if data_type == mstype.bfloat16:
|
|
2725
|
+
split_param = Parameter(Tensor(split_tensor, mstype.bfloat16), param_name, requires_grad, layerwise_parallel)
|
|
2726
|
+
else:
|
|
2727
|
+
split_param = Parameter(split_tensor, param_name, requires_grad, layerwise_parallel)
|
|
2642
2728
|
return split_param
|
|
2643
2729
|
|
|
2644
2730
|
|
|
@@ -2647,7 +2733,7 @@ def _calculation_net_size(net):
|
|
|
2647
2733
|
data_total = 0
|
|
2648
2734
|
net_dict = net.parameters_dict()
|
|
2649
2735
|
for name in net_dict:
|
|
2650
|
-
data_total += sys.getsizeof(net_dict[name].data.
|
|
2736
|
+
data_total += sys.getsizeof(net_dict[name].data.get_bytes()) / 1024
|
|
2651
2737
|
|
|
2652
2738
|
return data_total
|
|
2653
2739
|
|
|
@@ -392,7 +392,7 @@ def _fill_image_summary(tag: str, np_value, summary_image, input_format='NCHW'):
|
|
|
392
392
|
else:
|
|
393
393
|
if max_value != min_value:
|
|
394
394
|
# Mapping the value to range [0, 255] linearly.
|
|
395
|
-
scale_factor = 255/(max_value - min_value + 1)
|
|
395
|
+
scale_factor = 255 / (max_value - min_value + 1)
|
|
396
396
|
shift = min_value
|
|
397
397
|
tensor = tensor.astype(np.float32)
|
|
398
398
|
tensor = ((tensor - shift) * scale_factor).astype(np.uint8)
|
|
@@ -21,6 +21,7 @@ import re
|
|
|
21
21
|
import threading
|
|
22
22
|
import time
|
|
23
23
|
from collections import defaultdict
|
|
24
|
+
import numpy as np
|
|
24
25
|
|
|
25
26
|
from mindspore import log as logger
|
|
26
27
|
from mindspore.nn import Cell
|
|
@@ -36,6 +37,7 @@ from mindspore.train.summary._summary_adapter import get_event_file_name, packag
|
|
|
36
37
|
from mindspore.train.summary._writer_pool import WriterPool
|
|
37
38
|
from mindspore.train.summary.enums import PluginEnum
|
|
38
39
|
from mindspore.ops.operations import debug_ops
|
|
40
|
+
import mindspore.ops as ops
|
|
39
41
|
|
|
40
42
|
# for the moment, this lock is for caution's sake,
|
|
41
43
|
# there are actually no any concurrences happening.
|
|
@@ -59,12 +61,19 @@ def _cache_summary_tensor_data(summary):
|
|
|
59
61
|
with _summary_lock:
|
|
60
62
|
for item in summary:
|
|
61
63
|
SUMMARY_TENSOR_CACHE[item['name']] = item['data']
|
|
62
|
-
|
|
64
|
+
return True
|
|
63
65
|
|
|
64
66
|
|
|
65
|
-
def _get_summary_tensor_data():
|
|
67
|
+
def _get_summary_tensor_data(end_flag=None, del_end_flag=False):
|
|
66
68
|
"""Get summary tensor data."""
|
|
67
69
|
global SUMMARY_TENSOR_CACHE
|
|
70
|
+
if end_flag:
|
|
71
|
+
for _ in range(0, 100):
|
|
72
|
+
if SUMMARY_TENSOR_CACHE.get(end_flag):
|
|
73
|
+
break
|
|
74
|
+
time.sleep(0.01)
|
|
75
|
+
if del_end_flag and SUMMARY_TENSOR_CACHE.get(end_flag):
|
|
76
|
+
del SUMMARY_TENSOR_CACHE[end_flag]
|
|
68
77
|
with _summary_lock:
|
|
69
78
|
data = SUMMARY_TENSOR_CACHE
|
|
70
79
|
SUMMARY_TENSOR_CACHE = {}
|
|
@@ -89,7 +98,7 @@ def _record_summary_tensor_data():
|
|
|
89
98
|
"data": data[2]
|
|
90
99
|
}
|
|
91
100
|
summary_list.append(summary_value)
|
|
92
|
-
_cache_summary_tensor_data(summary_list)
|
|
101
|
+
_ = _cache_summary_tensor_data(summary_list)
|
|
93
102
|
debug_ops.SUMMARY_TENSOR_CACHE = []
|
|
94
103
|
|
|
95
104
|
|
|
@@ -209,6 +218,8 @@ class SummaryRecord:
|
|
|
209
218
|
self._num_process = num_process
|
|
210
219
|
self.raise_exception = raise_exception
|
|
211
220
|
self._export_options = export_options
|
|
221
|
+
self.tensor_summary = ops.TensorSummary()
|
|
222
|
+
|
|
212
223
|
try:
|
|
213
224
|
self._initialize()
|
|
214
225
|
except (TypeError, ValueError) as err:
|
|
@@ -389,7 +400,9 @@ class SummaryRecord:
|
|
|
389
400
|
return True
|
|
390
401
|
|
|
391
402
|
if self._mode == 'train':
|
|
392
|
-
|
|
403
|
+
step_end_flag = Tensor((np.ones([1])).astype(np.int32))
|
|
404
|
+
self.tensor_summary("step_end_flag_" + str(step), step_end_flag)
|
|
405
|
+
self._add_summary_tensor_data(step)
|
|
393
406
|
|
|
394
407
|
if not plugin_filter:
|
|
395
408
|
self._event_writer.write(self._consume_data_pool(step))
|
|
@@ -441,13 +454,16 @@ class SummaryRecord:
|
|
|
441
454
|
self._num_process,
|
|
442
455
|
self.raise_exception,
|
|
443
456
|
**filename_dict)
|
|
444
|
-
_get_summary_tensor_data()
|
|
457
|
+
_ = _get_summary_tensor_data()
|
|
445
458
|
atexit.register(self.close)
|
|
446
459
|
|
|
447
|
-
def _add_summary_tensor_data(self):
|
|
460
|
+
def _add_summary_tensor_data(self, step_index=-1):
|
|
448
461
|
"""Add summary tensor data."""
|
|
449
462
|
_record_summary_tensor_data()
|
|
450
|
-
|
|
463
|
+
end_flag = None
|
|
464
|
+
if step_index >= 0:
|
|
465
|
+
end_flag = "step_end_flag_" + str(step_index) + "[:Tensor]"
|
|
466
|
+
summary_data = _get_summary_tensor_data(end_flag=end_flag, del_end_flag=True)
|
|
451
467
|
if not summary_data:
|
|
452
468
|
logger.debug(f'No summary data bubbled from the network.')
|
|
453
469
|
for name, tensor in summary_data.items():
|
mindspore/version.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
__version__ = '2.2.
|
|
1
|
+
__version__ = '2.2.11'
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: mindspore
|
|
3
|
-
Version: 2.2.
|
|
3
|
+
Version: 2.2.11
|
|
4
4
|
Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
|
|
5
5
|
Home-page: https://www.mindspore.cn
|
|
6
6
|
Download-URL: https://github.com/mindspore-ai/mindspore/tags
|
|
@@ -318,10 +318,11 @@ Project stable branches will be in one of the following states:
|
|
|
318
318
|
|
|
319
319
|
| **Branch** | **Status** | **Initial Release Date** | **Next Phase** | **EOL Date**|
|
|
320
320
|
|------------|--------------|--------------------------|----------------------------------------|-------------|
|
|
321
|
+
| **r2.2** | Maintained | 2023-10-18 | Unmaintained <br> 2024-10-18 estimated | |
|
|
321
322
|
| **r2.1** | Maintained | 2023-07-29 | Unmaintained <br> 2024-07-29 estimated | |
|
|
322
323
|
| **r2.0** | Maintained | 2023-06-15 | Unmaintained <br> 2024-06-15 estimated | |
|
|
323
324
|
| **r1.10** | Maintained | 2023-02-02 | Unmaintained <br> 2024-02-02 estimated | |
|
|
324
|
-
| **r1.9** |
|
|
325
|
+
| **r1.9** | End Of Life | 2022-10-26 | | 2023-10-26 |
|
|
325
326
|
| **r1.8** | End Of Life | 2022-07-29 | | 2023-07-29 |
|
|
326
327
|
| **r1.7** | End Of Life | 2022-04-29 | | 2023-04-29 |
|
|
327
328
|
| **r1.6** | End Of Life | 2022-01-29 | | 2023-01-29 |
|