mindspore 2.2.0__cp37-cp37m-manylinux1_x86_64.whl → 2.2.11__cp37-cp37m-manylinux1_x86_64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. mindspore/.commit_id +1 -1
  2. mindspore/_akg/akg/composite/build_module.py +104 -20
  3. mindspore/_akg/akg/utils/ascend_profilier/cann_file_parser.py +76 -0
  4. mindspore/_akg/akg/utils/ascend_profilier/file_manager.py +56 -0
  5. mindspore/_akg/akg/utils/ascend_profilier/op_summary_bean.py +23 -0
  6. mindspore/_akg/akg/utils/ascend_profilier/op_summary_headers.py +8 -0
  7. mindspore/_akg/akg/utils/ascend_profilier/op_summary_parser.py +42 -0
  8. mindspore/_akg/akg/utils/ascend_profilier/path_manager.py +65 -0
  9. mindspore/_akg/akg/utils/composite_op_helper.py +7 -2
  10. mindspore/_akg/akg/utils/dump_ascend_meta.py +22 -3
  11. mindspore/_akg/akg/utils/kernel_exec.py +41 -15
  12. mindspore/_akg/akg/utils/tbe_codegen_utils.py +27 -6
  13. mindspore/_akg/akg/utils/util.py +56 -1
  14. mindspore/_c_dataengine.cpython-37m-x86_64-linux-gnu.so +0 -0
  15. mindspore/_c_expression.cpython-37m-x86_64-linux-gnu.so +0 -0
  16. mindspore/_checkparam.py +3 -3
  17. mindspore/_extends/graph_kernel/model/graph_split.py +84 -76
  18. mindspore/_extends/graph_kernel/splitter.py +3 -2
  19. mindspore/_extends/parallel_compile/akg_compiler/build_tbe_kernel.py +83 -66
  20. mindspore/_extends/parallel_compile/akg_compiler/tbe_topi.py +4 -4
  21. mindspore/_extends/parallel_compile/akg_compiler/util.py +10 -7
  22. mindspore/_extends/parallel_compile/tbe_compiler/tbe_helper.py +2 -1
  23. mindspore/_extends/parse/__init__.py +3 -2
  24. mindspore/_extends/parse/parser.py +6 -1
  25. mindspore/_extends/parse/standard_method.py +14 -11
  26. mindspore/_extends/remote/kernel_build_server.py +2 -1
  27. mindspore/_mindspore_offline_debug.cpython-37m-x86_64-linux-gnu.so +0 -0
  28. mindspore/bin/cache_admin +0 -0
  29. mindspore/bin/cache_server +0 -0
  30. mindspore/common/_utils.py +16 -0
  31. mindspore/common/api.py +1 -1
  32. mindspore/common/auto_dynamic_shape.py +81 -85
  33. mindspore/common/dump.py +1 -1
  34. mindspore/common/tensor.py +3 -20
  35. mindspore/config/op_info.config +1 -1
  36. mindspore/context.py +11 -4
  37. mindspore/dataset/engine/cache_client.py +8 -5
  38. mindspore/dataset/engine/datasets_standard_format.py +5 -0
  39. mindspore/dataset/vision/transforms.py +21 -21
  40. mindspore/experimental/optim/adam.py +1 -1
  41. mindspore/gen_ops.py +1 -1
  42. mindspore/include/api/model.h +17 -0
  43. mindspore/include/api/status.h +8 -3
  44. mindspore/lib/libdnnl.so.2 +0 -0
  45. mindspore/lib/libmindspore.so +0 -0
  46. mindspore/lib/libmindspore_backend.so +0 -0
  47. mindspore/lib/libmindspore_common.so +0 -0
  48. mindspore/lib/libmindspore_core.so +0 -0
  49. mindspore/lib/libmindspore_glog.so.0 +0 -0
  50. mindspore/lib/libmindspore_gpr.so.15 +0 -0
  51. mindspore/lib/libmindspore_grpc++.so.1 +0 -0
  52. mindspore/lib/libmindspore_grpc.so.15 +0 -0
  53. mindspore/lib/libmindspore_shared_lib.so +0 -0
  54. mindspore/lib/libnnacl.so +0 -0
  55. mindspore/lib/libopencv_core.so.4.5 +0 -0
  56. mindspore/lib/libopencv_imgcodecs.so.4.5 +0 -0
  57. mindspore/lib/libopencv_imgproc.so.4.5 +0 -0
  58. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend310/aic-ascend310-ops-info.json +123 -0
  59. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend310p/aic-ascend310p-ops-info.json +123 -0
  60. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend910/aic-ascend910-ops-info.json +158 -0
  61. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/config/ascend910b/aic-ascend910b-ops-info.json +37 -0
  62. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/add_dsl.py +46 -0
  63. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/add_tik.py +51 -0
  64. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/kv_cache_mgr.py +241 -0
  65. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/ai_core/tbe/custom_aicore_ops_impl/matmul_tik.py +212 -0
  66. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/vector_core/tbe/custom_aicore_ops_impl/add_dsl.py +46 -0
  67. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/vector_core/tbe/custom_aicore_ops_impl/add_tik.py +51 -0
  68. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/vector_core/tbe/custom_aicore_ops_impl/kv_cache_mgr.py +241 -0
  69. mindspore/lib/plugin/ascend/custom_aicore_ops/op_impl/vector_core/tbe/custom_aicore_ops_impl/matmul_tik.py +212 -0
  70. mindspore/lib/plugin/ascend/custom_aicore_ops/op_proto/libop_proto.so +0 -0
  71. mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_aicpu_kernels.so +0 -0
  72. mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/aicpu_kernel/impl/libcust_cpu_kernels.so +0 -0
  73. mindspore/lib/plugin/ascend/custom_aicpu_ops/op_impl/cpu/config/cust_aicpu_kernel.json +78 -80
  74. mindspore/lib/plugin/ascend/custom_aicpu_ops/op_proto/libcust_op_proto.so +0 -0
  75. mindspore/lib/plugin/ascend/libakg.so +0 -0
  76. mindspore/lib/plugin/ascend/libhccl_plugin.so +0 -0
  77. mindspore/lib/plugin/ascend/libmindspore_aicpu_kernels.so +0 -0
  78. mindspore/lib/plugin/ascend/libmindspore_cpu_kernels.so +0 -0
  79. mindspore/lib/plugin/cpu/libakg.so +0 -0
  80. mindspore/lib/plugin/gpu/libcuda_ops.so.10 +0 -0
  81. mindspore/lib/plugin/gpu/libcuda_ops.so.11 +0 -0
  82. mindspore/lib/plugin/gpu10.1/libakg.so +0 -0
  83. mindspore/lib/plugin/gpu10.1/libnccl.so.2 +0 -0
  84. mindspore/lib/plugin/gpu11.1/libakg.so +0 -0
  85. mindspore/lib/plugin/gpu11.6/libakg.so +0 -0
  86. mindspore/lib/plugin/gpu11.6/libnccl.so.2 +0 -0
  87. mindspore/lib/plugin/libmindspore_ascend.so.1 +0 -0
  88. mindspore/lib/plugin/libmindspore_ascend.so.2 +0 -0
  89. mindspore/lib/plugin/libmindspore_gpu.so.10.1 +0 -0
  90. mindspore/lib/plugin/libmindspore_gpu.so.11.1 +0 -0
  91. mindspore/lib/plugin/libmindspore_gpu.so.11.6 +0 -0
  92. mindspore/nn/cell.py +0 -3
  93. mindspore/nn/layer/activation.py +4 -5
  94. mindspore/nn/layer/conv.py +39 -23
  95. mindspore/nn/layer/flash_attention.py +54 -129
  96. mindspore/nn/layer/math.py +3 -7
  97. mindspore/nn/layer/rnn_cells.py +5 -5
  98. mindspore/nn/wrap/__init__.py +4 -2
  99. mindspore/nn/wrap/cell_wrapper.py +12 -3
  100. mindspore/numpy/utils_const.py +5 -5
  101. mindspore/ops/_grad_experimental/grad_array_ops.py +1 -1
  102. mindspore/ops/_grad_experimental/grad_implementations.py +2 -2
  103. mindspore/ops/_grad_experimental/grad_math_ops.py +19 -18
  104. mindspore/ops/_grad_experimental/grad_sparse_ops.py +3 -3
  105. mindspore/ops/_op_impl/aicpu/add.py +3 -3
  106. mindspore/ops/_op_impl/aicpu/linear_sum_assignment.py +21 -2
  107. mindspore/ops/_utils/utils.py +2 -0
  108. mindspore/ops/composite/multitype_ops/_compile_utils.py +2 -1
  109. mindspore/ops/composite/multitype_ops/getitem_impl.py +2 -2
  110. mindspore/ops/function/array_func.py +10 -7
  111. mindspore/ops/function/grad/grad_func.py +0 -1
  112. mindspore/ops/function/nn_func.py +98 -9
  113. mindspore/ops/function/random_func.py +2 -1
  114. mindspore/ops/op_info_register.py +24 -21
  115. mindspore/ops/operations/__init__.py +6 -2
  116. mindspore/ops/operations/_grad_ops.py +25 -6
  117. mindspore/ops/operations/_inner_ops.py +155 -23
  118. mindspore/ops/operations/array_ops.py +9 -7
  119. mindspore/ops/operations/comm_ops.py +2 -2
  120. mindspore/ops/operations/custom_ops.py +85 -68
  121. mindspore/ops/operations/inner_ops.py +26 -3
  122. mindspore/ops/operations/math_ops.py +7 -6
  123. mindspore/ops/operations/nn_ops.py +193 -49
  124. mindspore/parallel/_parallel_serialization.py +10 -3
  125. mindspore/parallel/_tensor.py +4 -1
  126. mindspore/parallel/checkpoint_transform.py +13 -2
  127. mindspore/parallel/shard.py +17 -10
  128. mindspore/profiler/common/util.py +1 -0
  129. mindspore/profiler/parser/ascend_hccl_generator.py +232 -0
  130. mindspore/profiler/parser/ascend_msprof_exporter.py +86 -43
  131. mindspore/profiler/parser/ascend_msprof_generator.py +196 -9
  132. mindspore/profiler/parser/ascend_op_generator.py +1 -1
  133. mindspore/profiler/parser/ascend_timeline_generator.py +6 -182
  134. mindspore/profiler/parser/base_timeline_generator.py +1 -1
  135. mindspore/profiler/parser/cpu_gpu_timeline_generator.py +2 -2
  136. mindspore/profiler/parser/framework_parser.py +1 -1
  137. mindspore/profiler/parser/profiler_info.py +19 -0
  138. mindspore/profiler/profiling.py +46 -24
  139. mindspore/rewrite/api/pattern_engine.py +1 -1
  140. mindspore/rewrite/parsers/for_parser.py +7 -7
  141. mindspore/rewrite/parsers/module_parser.py +4 -4
  142. mindspore/rewrite/symbol_tree.py +1 -4
  143. mindspore/run_check/_check_version.py +5 -3
  144. mindspore/safeguard/rewrite_obfuscation.py +52 -28
  145. mindspore/scipy/ops.py +55 -5
  146. mindspore/scipy/optimize/__init__.py +3 -2
  147. mindspore/scipy/optimize/linear_sum_assignment.py +38 -33
  148. mindspore/train/callback/_summary_collector.py +1 -1
  149. mindspore/train/dataset_helper.py +1 -0
  150. mindspore/train/model.py +2 -2
  151. mindspore/train/serialization.py +97 -11
  152. mindspore/train/summary/_summary_adapter.py +1 -1
  153. mindspore/train/summary/summary_record.py +23 -7
  154. mindspore/version.py +1 -1
  155. {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/METADATA +3 -2
  156. {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/RECORD +160 -151
  157. mindspore/ops/_op_impl/_custom_op/flash_attention/attention.py +0 -406
  158. mindspore/ops/_op_impl/_custom_op/flash_attention/constants.py +0 -41
  159. mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_bwd.py +0 -467
  160. mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_fwd.py +0 -563
  161. mindspore/ops/_op_impl/_custom_op/flash_attention/flash_attention_impl.py +0 -193
  162. mindspore/ops/_op_impl/_custom_op/flash_attention/tik_ops_utils.py +0 -435
  163. mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/__init__.py +0 -0
  164. mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/sparse_tiling.py +0 -45
  165. mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/strategy.py +0 -67
  166. mindspore/ops/_op_impl/_custom_op/flash_attention/tiling_strategy/wukong_tiling.py +0 -62
  167. /mindspore/{ops/_op_impl/_custom_op/flash_attention → _akg/akg/utils/ascend_profilier}/__init__.py +0 -0
  168. {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/WHEEL +0 -0
  169. {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/entry_points.txt +0 -0
  170. {mindspore-2.2.0.dist-info → mindspore-2.2.11.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- # Copyright 2021 Huawei Technologies Co., Ltd
1
+ # Copyright 2021-2023 Huawei Technologies Co., Ltd
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -15,5 +15,6 @@
15
15
  """Optimize submodule"""
16
16
  from .minimize import minimize
17
17
  from .line_search import line_search
18
+ from .linear_sum_assignment import linear_sum_assignment
18
19
 
19
- __all__ = ["minimize", "line_search"]
20
+ __all__ = ["minimize", "line_search", "linear_sum_assignment"]
@@ -1,4 +1,4 @@
1
- # Copyright 2022 Huawei Technologies Co., Ltd
1
+ # Copyright 2023 Huawei Technologies Co., Ltd
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -15,59 +15,64 @@
15
15
  """Linear Sum Assignment"""
16
16
  import sys
17
17
  from ..ops import LinearSumAssignment
18
- from ...common import dtype as mstype
19
- from ..utils import _mstype_check, _dtype_check
20
18
  from ... import Tensor
21
19
 
22
20
 
23
- def _linear_sum_assignment(cost_matrix, maximize, dimension_limit=Tensor(sys.maxsize)):
24
- """
21
+ def linear_sum_assignment(cost_matrix, maximize, dimension_limit=Tensor(sys.maxsize)):
22
+ r"""
25
23
  Solve the linear sum assignment problem.
26
24
 
25
+ The assignment problem is represented as follows:
26
+
27
+ .. math::
28
+ min\sum_{i}^{} \sum_{j}^{} C_{i,j} X_{i,j}
29
+
30
+ where :math:`C` is cost matrix, :math:`X_{i,j} = 1` means column :math:`j` is assigned to row :math:`i` .
31
+
27
32
  Args:
28
- cost_matrix: 2-D Input Tensor.
29
- The cost matrix of the bipartite graph.
30
- maximize: 0-D Input bool Tensor.
31
- Calculates a maximum weight matching if true.
32
- dimension_limit: 0-D Input Tensor.
33
- A scalar used to limit the actual size of the 1st dimension. Optimized for
34
- padding scenes. Default means no dimension limit.
33
+ cost_matrix (Tensor): 2-D cost matrix. Tensor of shape :math:`(M, N)` .
34
+ maximize (bool): Calculate a maximum weight matching if true, otherwise calculate a minimum weight matching.
35
+ dimension_limit (Tensor, optional): A scalar used to limit the actual size of the 2nd dimension of
36
+ ``cost_matrix``. Default is ``Tensor(sys.maxsize)``, which means no limitation. The type is 0-D int64
37
+ Tensor.
35
38
 
36
39
  Returns:
37
- 1-D Output Tensors with 'row_indx' and 'col_idx'. An array of row indices and
38
- one of corresponding column indices giving the optimal assignment. If specified
39
- dimension_limit, padding value at the end would be -1.
40
+ A tuple of tensors containing 'row_idx' and 'col_idx'.
41
+
42
+ - **row_idx** (Tensor) - Row indices of the problem. If `dimension_limit` is given, -1 would be padded at the
43
+ end. The shape is :math:`(N, )` , where :math:`N` is the minimum value of `cost_matrix` dimension.
44
+ - **col_idx** (Tensor) - Column indices of the problem. If `dimension_limit` is given, -1 would be padded at
45
+ the end. The shape is :math:`(N, )` , where :math:`N` is the minimum value of `cost_matrix` dimension.
46
+
47
+ Raises:
48
+ TypeError: If the data type of `cost_matrix` is not the type in [float16, float32, float64,
49
+ int8, int16, int32, int64, uint8, uint16, uint32, uint64, bool]
50
+ TypeError: If the type of `maximize` is not bool.
51
+ TypeError: If the data type of `dimension_limit` is not int64.
52
+ ValueError: If the rank of `cost_matrix` is not 2.
53
+
40
54
 
41
55
  Supported Platforms:
42
- ``CPU``
56
+ ``Ascend`` ``CPU``
43
57
 
44
- Examples:
58
+ Examples:
45
59
  >>> import mindspore as ms
46
60
  >>> import numpy as np
47
61
  >>> from mindspore import Tensor
48
- >>> from mindspore.scipy.optimize.linear_sum_assignment import _linear_sum_assignment as lsap
49
- >>> cost_matrix = Tensor(np.array([[2, 3, 3], [3, 2, 3], [3, 3, 2]])).astype("float64")
62
+ >>> import mindspore.scipy.optimize.linear_sum_assignment as lsap
63
+ >>> cost_matrix = Tensor(np.array([[2, 3, 3], [3, 2, 3], [3, 3, 2]])).astype(ms.float64)
50
64
  >>> dimension_limit = Tensor(2)
51
- >>> maximize = Tensor(False)
65
+ >>> maximize = False
52
66
  >>> a, b = lsap(cost_matrix, maximize, dimension_limit)
53
67
  >>> print(a)
54
- [[0 1 -1]]
68
+ [0 1 -1]
55
69
  >>> print(b)
56
- [[0 1 -1]]
70
+ [0 1 -1]
57
71
  >>> a, b = lsap(cost_matrix, maximize)
58
72
  >>> print(a)
59
- [[0 1 2]]
73
+ [0 1 2]
60
74
  >>> print(b)
61
- [[0 1 2]]
75
+ [0 1 2]
62
76
  """
63
- func_name = 'linear_sum_assignment'
64
- _mstype_check(func_name, cost_matrix, mstype.TensorType, 'cost_matrix')
65
- _mstype_check(func_name, dimension_limit,
66
- mstype.TensorType, 'dimension_limit')
67
- _mstype_check(func_name, maximize, mstype.TensorType, 'maximize')
68
- _dtype_check(func_name, cost_matrix, [mstype.float32, mstype.float64])
69
- _dtype_check(func_name, dimension_limit, [mstype.int64])
70
- _dtype_check(func_name, maximize, [mstype.bool_])
71
-
72
77
  solve = LinearSumAssignment()
73
78
  return solve(cost_matrix, dimension_limit, maximize)
@@ -1157,7 +1157,7 @@ class SummaryCollector(Callback):
1157
1157
  except TypeError as exc:
1158
1158
  logger.warning("Summary cannot collect the type of metrics, currently support type: dict, list, tuple, "
1159
1159
  "str, int, float, bool and None. %s.", str(exc))
1160
- self._parse_dataset(cb_params, eval_lineage)
1160
+ _ = self._parse_dataset(cb_params, eval_lineage)
1161
1161
 
1162
1162
  eval_lineage_message = self._package_eval_lineage_message(eval_lineage)
1163
1163
  self._record.add_value(PluginEnum.EVAL_LINEAGE.value, 'eval_lineage', eval_lineage_message)
@@ -52,6 +52,7 @@ def _dynamic_sink_data(dataset, dataset_iter):
52
52
  if hasattr(dataset_iter, "sink_size") and \
53
53
  dataset_iter.sink_size == 1 and \
54
54
  dataset.get_dataset_size() != 1 and \
55
+ not hasattr(dataset, "__no_send__") and \
55
56
  hasattr(dataset_iter, "sink_count") and \
56
57
  dataset_iter.sink_count == 1:
57
58
  return True
mindspore/train/model.py CHANGED
@@ -1586,7 +1586,7 @@ class Model:
1586
1586
  use_past = not is_first_iteration
1587
1587
  model_group_id = self._mindspore_lite_model_group_id
1588
1588
 
1589
- check_input_data(*predict_data, data_class=Tensor)
1589
+ check_input_data(*predict_data, data_class=(int, float, str, None, Tensor))
1590
1590
  if use_past:
1591
1591
  # Execute incremental model inference
1592
1592
  if not self._lite_incremental_predictor:
@@ -1867,7 +1867,7 @@ class Model:
1867
1867
  if _get_parallel_mode() not in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL):
1868
1868
  raise RuntimeError('Infer predict layout only supports semi auto parallel and auto parallel mode.')
1869
1869
  _parallel_predict_check()
1870
- check_input_data(*predict_data, data_class=Tensor)
1870
+ check_input_data(*predict_data, data_class=(int, float, str, None, Tensor))
1871
1871
 
1872
1872
  predict_net = self._predict_network
1873
1873
  # Unlike the cases in build_train_network() and build_eval_network(), 'multi_subgraphs' is not set
@@ -65,15 +65,17 @@ from mindspore.train._utils import read_proto
65
65
  from mindspore._c_expression import load_mindir, _encrypt, _decrypt, _is_cipher_file, dynamic_obfuscate_mindir, \
66
66
  split_mindir, split_dynamic_mindir
67
67
  from ..ops.operations._opaque_predicate_registry import add_opaque_predicate, clean_funcs
68
+ from ..ops.operations import Cast
68
69
 
69
70
  tensor_to_ms_type = {"Int8": mstype.int8, "UInt8": mstype.uint8, "Int16": mstype.int16, "UInt16": mstype.uint16,
70
71
  "Int32": mstype.int32, "UInt32": mstype.uint32, "Int64": mstype.int64, "UInt64": mstype.uint64,
71
72
  "Float16": mstype.float16, "Float32": mstype.float32, "Float64": mstype.float64,
72
- "Bool": mstype.bool_, "str": mstype.string}
73
+ "Bool": mstype.bool_, "str": mstype.string, "BFloat16": mstype.bfloat16}
73
74
 
74
75
  tensor_to_np_type = {"Int8": np.int8, "UInt8": np.uint8, "Int16": np.int16, "UInt16": np.uint16,
75
76
  "Int32": np.int32, "UInt32": np.uint32, "Int64": np.int64, "UInt64": np.uint64,
76
- "Float16": np.float16, "Float32": np.float32, "Float64": np.float64, "Bool": np.bool_, "str": "U"}
77
+ "Float16": np.float16, "Float32": np.float32, "Float64": np.float64, "Bool": np.bool_, "str": "U",
78
+ "BFloat16": np.float32}
77
79
 
78
80
  np_type_convert = {"int32": np.int32, "float32": np.float32, "float16": np.float16, "float64": np.float64}
79
81
 
@@ -91,6 +93,8 @@ PARAMETER_SPLIT_SIZE = 1024 * 1024 * 1024
91
93
  ENCRYPT_BLOCK_SIZE = 64 * 1024
92
94
  INT_64_MAX = 9223372036854775807
93
95
 
96
+ cpu_cast = Cast().set_device("CPU")
97
+
94
98
 
95
99
  def _special_process_par(par, new_par):
96
100
  """
@@ -107,7 +111,11 @@ def _special_process_par(par, new_par):
107
111
  if new_par.data.shape[par_shape_len + i] != 1:
108
112
  return False
109
113
 
110
- new_val = new_par.data.asnumpy()
114
+ if new_par.data.dtype == mstype.bfloat16:
115
+ new_val = cpu_cast(new_par.data, mstype.float32).asnumpy()
116
+ else:
117
+ new_val = new_par.data.asnumpy()
118
+
111
119
  new_val = new_val.reshape(par.data.shape)
112
120
  par.set_data(Tensor(new_val, par.data.dtype))
113
121
  return True
@@ -128,7 +136,10 @@ def _update_param(param, new_param, strict_load):
128
136
 
129
137
  if param.data.dtype != new_param.data.dtype:
130
138
  if _type_convert(param, new_param, strict_load):
131
- new_tensor = Tensor(new_param.data.asnumpy(), param.data.dtype)
139
+ if new_param.data.dtype == mstype.bfloat16:
140
+ new_tensor = cpu_cast(new_param.data, param.data.dtype)
141
+ else:
142
+ new_tensor = Tensor(new_param.data.asnumpy(), param.data.dtype)
132
143
  param.set_data(new_tensor, param.sliced)
133
144
  return
134
145
 
@@ -231,10 +242,16 @@ def _exec_save(ckpt_file_name, data_list, enc_key=None, enc_mode="AES-GCM", map_
231
242
  continue
232
243
  if value[0] == "offload_parameter":
233
244
  new_value = value[1:]
234
- new_value[2] = value[3].asnumpy().reshape(-1)
245
+ if value[3].dtype == mstype.bfloat16:
246
+ new_value[2] = cpu_cast(value[3], mstype.float32).asnumpy().reshape(-1)
247
+ else:
248
+ new_value[2] = value[3].asnumpy().reshape(-1)
235
249
  _write_parameter_data(name, new_value, f, enc_key, plain_data)
236
250
  _offload_if_config(value[3])
237
251
  continue
252
+ if value[0] == "BFloat16_tensor":
253
+ _write_bfloat16_data(name, value, f, enc_key, plain_data)
254
+ continue
238
255
  if isinstance(value[2], Tensor):
239
256
  _write_hugeparameter(name, value, f)
240
257
  continue
@@ -269,6 +286,21 @@ def _write_random_seed(name, value, f):
269
286
  f.write(checkpoint_list.SerializeToString())
270
287
 
271
288
 
289
+ def _write_bfloat16_data(name, value, f, enc_key, plain_data):
290
+ """Write bfloat16 data into protobuf file"""
291
+ checkpoint_list = Checkpoint()
292
+ param_value = checkpoint_list.value.add()
293
+ param_value.tag = name
294
+ param_tensor = param_value.tensor
295
+ param_tensor.dims.extend(value[1])
296
+ param_tensor.tensor_type = value[2]
297
+ param_tensor.tensor_content = value[3].get_bytes()
298
+ if enc_key is None:
299
+ f.write(checkpoint_list.SerializeToString())
300
+ else:
301
+ plain_data.write(checkpoint_list.SerializeToString())
302
+
303
+
272
304
  def _write_parameter_data(name, value, f, enc_key, plain_data):
273
305
  """Write parameter data into protobuf file."""
274
306
  data_size = value[2].nbytes / 1024
@@ -447,6 +479,10 @@ def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
447
479
  elif param["data"][0] == "offload_parameter":
448
480
  data_list[key].append("offload_parameter")
449
481
  _save_param_list_data(data_list, key, param)
482
+ elif param["data"][0] == "BFloat16_tensor":
483
+ data_list[key].append("BFloat16_tensor")
484
+ _save_param_list_data(data_list, key, param)
485
+ continue
450
486
 
451
487
  if isinstance(param["data"], str):
452
488
  data_list[key].append([0])
@@ -456,6 +492,15 @@ def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
456
492
  else:
457
493
  if isinstance(param["data"], Parameter):
458
494
  param["data"].init_data()
495
+ if isinstance(param["data"], Tensor) and param["data"].dtype == mstype.bfloat16:
496
+ data_list[key].append("BFloat16_tensor")
497
+ dims = []
498
+ for dim in param["data"].shape:
499
+ dims.append(dim)
500
+ data_list[key].append(dims)
501
+ data_list[key].append("BFloat16")
502
+ data_list[key].append(cpu_cast(param["data"], mstype.float32))
503
+ continue
459
504
  dims = []
460
505
  if param['data'].shape == ():
461
506
  dims.append(0)
@@ -465,7 +510,10 @@ def save_checkpoint(save_obj, ckpt_file_name, integrated_save=True,
465
510
  data_list[key].append(dims)
466
511
  tensor_type = str(param["data"].dtype)
467
512
  data_list[key].append(tensor_type)
468
- data = param["data"].asnumpy().reshape(-1)
513
+ if param["data"].dtype == mstype.bfloat16:
514
+ data = cpu_cast(param["data"], mstype.float32).asnumpy().reshape(-1)
515
+ else:
516
+ data = param["data"].asnumpy().reshape(-1)
469
517
  data_list[key].append(data)
470
518
 
471
519
  if async_save:
@@ -571,6 +619,12 @@ def _convert_cell_to_param_list(save_obj, integrated_save, append_dict, choice_f
571
619
  param_data.append(param_tensor.shape)
572
620
  param_data.append(str(param_tensor.dtype))
573
621
  param_data.append(value.key)
622
+ elif value.data.dtype == mstype.bfloat16:
623
+ param_data = ["BFloat16_tensor"]
624
+ param_data.append(cpu_cast(value.data, mstype.float32))
625
+ param_data.append(value.data.shape)
626
+ param_data.append("BFloat16")
627
+ param_data.append(value.key)
574
628
  else:
575
629
  param_data = Tensor(value.data.asnumpy())
576
630
 
@@ -1065,6 +1119,13 @@ def load_checkpoint(ckpt_file_name, net=None, strict_load=False, filter_prefix=N
1065
1119
  if data_type == 'str':
1066
1120
  str_length = int(len(data) / 4)
1067
1121
  np_type = np_type + str(str_length)
1122
+ if data_type == "BFloat16":
1123
+ dims = element.tensor.dims
1124
+ param_data = np.frombuffer(data, np_type)
1125
+ param_data = param_data.reshape(list(dims))
1126
+ parameter = Parameter(Tensor(param_data, ms_type), name=element.tag)
1127
+ parameter_dict[element.tag] = parameter
1128
+ continue
1068
1129
  element_data = np.frombuffer(data, np_type)
1069
1130
  param_data_list.append(element_data)
1070
1131
  if (element_id == len(checkpoint_list.value) - 1) or \
@@ -1806,8 +1867,15 @@ def _split_save(net_dict, model, file_name, is_encrypt, **kwargs):
1806
1867
  data_file_name = os.path.join(dirname, external_local)
1807
1868
  f, parameter_size, offset = _get_data_file(is_encrypt, kwargs, data_file_name)
1808
1869
  try:
1870
+ round_ = 0
1871
+ names = []
1809
1872
  for param_proto in model.graph.parameter:
1810
1873
  name = param_proto.name[param_proto.name.find(":") + 1:]
1874
+ names.append((name, param_proto))
1875
+ names.sort(key=lambda x: x[0])
1876
+ for pairs in names:
1877
+ name = pairs[0]
1878
+ param_proto = pairs[1]
1811
1879
  param = net_dict[name]
1812
1880
  raw_data = param.data.get_bytes()
1813
1881
  data_length = len(raw_data)
@@ -1827,6 +1895,8 @@ def _split_save(net_dict, model, file_name, is_encrypt, **kwargs):
1827
1895
  offset += (data_length + append_size)
1828
1896
  write_data = _encrypt_data(is_encrypt, write_data, kwargs)
1829
1897
  f.write(write_data)
1898
+ round_ += 1
1899
+ logger.debug(f"writing {round_}th split data, name:{name}")
1830
1900
 
1831
1901
  graph_file_name = os.path.join(dirname, file_prefix + "_graph.mindir")
1832
1902
  if os.path.exists(graph_file_name):
@@ -2315,7 +2385,12 @@ def merge_sliced_parameter(sliced_parameters, strategy=None):
2315
2385
 
2316
2386
  layerwise_parallel = sliced_parameters[0].layerwise_parallel
2317
2387
  requires_grad = sliced_parameters[0].requires_grad
2318
- sliced_data = [parameter.data.asnumpy() for parameter in sliced_parameters]
2388
+ sliced_data = []
2389
+ for parameter in sliced_parameters:
2390
+ if parameter.data.dtype == mstype.bfloat16:
2391
+ sliced_data.append(cpu_cast(parameter.data, mstype.float32).asnumpy())
2392
+ else:
2393
+ sliced_data.append(parameter.data.asnumpy())
2319
2394
 
2320
2395
  if not strategy:
2321
2396
  merged_tensor = Tensor(np.concatenate(sliced_data))
@@ -2521,7 +2596,11 @@ def load_distributed_checkpoint(network, checkpoint_filenames, predict_strategy=
2521
2596
  param_index = list(set(param_index))
2522
2597
  param_index.sort()
2523
2598
  for rank_num in param_index:
2524
- param_stride.append(param_total_dict[param.name][rank_num].data.asnumpy())
2599
+ if param_total_dict[param.name][rank_num].data.dtype == mstype.bfloat16:
2600
+ param_stride.append(
2601
+ cpu_cast(param_total_dict[param.name][rank_num].data, mstype.float32).asnumpy())
2602
+ else:
2603
+ param_stride.append(param_total_dict[param.name][rank_num].data.asnumpy())
2525
2604
 
2526
2605
  sliced_param = Parameter(Tensor(np.concatenate(param_stride)), name=param.name)
2527
2606
  else:
@@ -2536,7 +2615,10 @@ def load_distributed_checkpoint(network, checkpoint_filenames, predict_strategy=
2536
2615
  split_param = _merge_and_split(sliced_params, _param_unique_strategy, predict_strategy)
2537
2616
  opt_shard_group = predict_strategy[param.name][5] if predict_strategy else None
2538
2617
  if opt_shard_group:
2539
- data = split_param.data.asnumpy()
2618
+ if split_param.data.dtype == mstype.bfloat16:
2619
+ data = cpu_cast(split_param.data, mstype.float32).asnumpy()
2620
+ else:
2621
+ data = split_param.data.asnumpy()
2540
2622
  rank = get_rank(opt_shard_group)
2541
2623
  size = get_group_size(opt_shard_group)
2542
2624
  try:
@@ -2638,7 +2720,11 @@ def _merge_and_split(sliced_params, train_strategy, predict_strategy):
2638
2720
  split_tensor = _load_tensor(merged_param.data, tensor_layout[0], tensor_layout[1], rank)
2639
2721
  requires_grad = merged_param.requires_grad
2640
2722
  layerwise_parallel = merged_param.layerwise_parallel
2641
- split_param = Parameter(split_tensor, param_name, requires_grad, layerwise_parallel)
2723
+ data_type = merged_param.data.dtype
2724
+ if data_type == mstype.bfloat16:
2725
+ split_param = Parameter(Tensor(split_tensor, mstype.bfloat16), param_name, requires_grad, layerwise_parallel)
2726
+ else:
2727
+ split_param = Parameter(split_tensor, param_name, requires_grad, layerwise_parallel)
2642
2728
  return split_param
2643
2729
 
2644
2730
 
@@ -2647,7 +2733,7 @@ def _calculation_net_size(net):
2647
2733
  data_total = 0
2648
2734
  net_dict = net.parameters_dict()
2649
2735
  for name in net_dict:
2650
- data_total += sys.getsizeof(net_dict[name].data.asnumpy().tobytes()) / 1024
2736
+ data_total += sys.getsizeof(net_dict[name].data.get_bytes()) / 1024
2651
2737
 
2652
2738
  return data_total
2653
2739
 
@@ -392,7 +392,7 @@ def _fill_image_summary(tag: str, np_value, summary_image, input_format='NCHW'):
392
392
  else:
393
393
  if max_value != min_value:
394
394
  # Mapping the value to range [0, 255] linearly.
395
- scale_factor = 255/(max_value - min_value + 1)
395
+ scale_factor = 255 / (max_value - min_value + 1)
396
396
  shift = min_value
397
397
  tensor = tensor.astype(np.float32)
398
398
  tensor = ((tensor - shift) * scale_factor).astype(np.uint8)
@@ -21,6 +21,7 @@ import re
21
21
  import threading
22
22
  import time
23
23
  from collections import defaultdict
24
+ import numpy as np
24
25
 
25
26
  from mindspore import log as logger
26
27
  from mindspore.nn import Cell
@@ -36,6 +37,7 @@ from mindspore.train.summary._summary_adapter import get_event_file_name, packag
36
37
  from mindspore.train.summary._writer_pool import WriterPool
37
38
  from mindspore.train.summary.enums import PluginEnum
38
39
  from mindspore.ops.operations import debug_ops
40
+ import mindspore.ops as ops
39
41
 
40
42
  # for the moment, this lock is for caution's sake,
41
43
  # there are actually no any concurrences happening.
@@ -59,12 +61,19 @@ def _cache_summary_tensor_data(summary):
59
61
  with _summary_lock:
60
62
  for item in summary:
61
63
  SUMMARY_TENSOR_CACHE[item['name']] = item['data']
62
- return True
64
+ return True
63
65
 
64
66
 
65
- def _get_summary_tensor_data():
67
+ def _get_summary_tensor_data(end_flag=None, del_end_flag=False):
66
68
  """Get summary tensor data."""
67
69
  global SUMMARY_TENSOR_CACHE
70
+ if end_flag:
71
+ for _ in range(0, 100):
72
+ if SUMMARY_TENSOR_CACHE.get(end_flag):
73
+ break
74
+ time.sleep(0.01)
75
+ if del_end_flag and SUMMARY_TENSOR_CACHE.get(end_flag):
76
+ del SUMMARY_TENSOR_CACHE[end_flag]
68
77
  with _summary_lock:
69
78
  data = SUMMARY_TENSOR_CACHE
70
79
  SUMMARY_TENSOR_CACHE = {}
@@ -89,7 +98,7 @@ def _record_summary_tensor_data():
89
98
  "data": data[2]
90
99
  }
91
100
  summary_list.append(summary_value)
92
- _cache_summary_tensor_data(summary_list)
101
+ _ = _cache_summary_tensor_data(summary_list)
93
102
  debug_ops.SUMMARY_TENSOR_CACHE = []
94
103
 
95
104
 
@@ -209,6 +218,8 @@ class SummaryRecord:
209
218
  self._num_process = num_process
210
219
  self.raise_exception = raise_exception
211
220
  self._export_options = export_options
221
+ self.tensor_summary = ops.TensorSummary()
222
+
212
223
  try:
213
224
  self._initialize()
214
225
  except (TypeError, ValueError) as err:
@@ -389,7 +400,9 @@ class SummaryRecord:
389
400
  return True
390
401
 
391
402
  if self._mode == 'train':
392
- self._add_summary_tensor_data()
403
+ step_end_flag = Tensor((np.ones([1])).astype(np.int32))
404
+ self.tensor_summary("step_end_flag_" + str(step), step_end_flag)
405
+ self._add_summary_tensor_data(step)
393
406
 
394
407
  if not plugin_filter:
395
408
  self._event_writer.write(self._consume_data_pool(step))
@@ -441,13 +454,16 @@ class SummaryRecord:
441
454
  self._num_process,
442
455
  self.raise_exception,
443
456
  **filename_dict)
444
- _get_summary_tensor_data()
457
+ _ = _get_summary_tensor_data()
445
458
  atexit.register(self.close)
446
459
 
447
- def _add_summary_tensor_data(self):
460
+ def _add_summary_tensor_data(self, step_index=-1):
448
461
  """Add summary tensor data."""
449
462
  _record_summary_tensor_data()
450
- summary_data = _get_summary_tensor_data()
463
+ end_flag = None
464
+ if step_index >= 0:
465
+ end_flag = "step_end_flag_" + str(step_index) + "[:Tensor]"
466
+ summary_data = _get_summary_tensor_data(end_flag=end_flag, del_end_flag=True)
451
467
  if not summary_data:
452
468
  logger.debug(f'No summary data bubbled from the network.')
453
469
  for name, tensor in summary_data.items():
mindspore/version.py CHANGED
@@ -1 +1 @@
1
- __version__ = '2.2.0'
1
+ __version__ = '2.2.11'
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: mindspore
3
- Version: 2.2.0
3
+ Version: 2.2.11
4
4
  Summary: MindSpore is a new open source deep learning training/inference framework that could be used for mobile, edge and cloud scenarios.
5
5
  Home-page: https://www.mindspore.cn
6
6
  Download-URL: https://github.com/mindspore-ai/mindspore/tags
@@ -318,10 +318,11 @@ Project stable branches will be in one of the following states:
318
318
 
319
319
  | **Branch** | **Status** | **Initial Release Date** | **Next Phase** | **EOL Date**|
320
320
  |------------|--------------|--------------------------|----------------------------------------|-------------|
321
+ | **r2.2** | Maintained | 2023-10-18 | Unmaintained <br> 2024-10-18 estimated | |
321
322
  | **r2.1** | Maintained | 2023-07-29 | Unmaintained <br> 2024-07-29 estimated | |
322
323
  | **r2.0** | Maintained | 2023-06-15 | Unmaintained <br> 2024-06-15 estimated | |
323
324
  | **r1.10** | Maintained | 2023-02-02 | Unmaintained <br> 2024-02-02 estimated | |
324
- | **r1.9** | Maintained | 2022-10-26 | Unmaintained <br> 2023-10-26 estimated | |
325
+ | **r1.9** | End Of Life | 2022-10-26 | | 2023-10-26 |
325
326
  | **r1.8** | End Of Life | 2022-07-29 | | 2023-07-29 |
326
327
  | **r1.7** | End Of Life | 2022-04-29 | | 2023-04-29 |
327
328
  | **r1.6** | End Of Life | 2022-01-29 | | 2023-01-29 |