mindspore 2.5.0__cp311-cp311-win_amd64.whl → 2.6.0rc1__cp311-cp311-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mindspore might be problematic. Click here for more details.
- mindspore/.commit_id +1 -1
- mindspore/Microsoft.VisualStudio.Telemetry.dll +0 -0
- mindspore/Newtonsoft.Json.dll +0 -0
- mindspore/__init__.py +6 -4
- mindspore/_c_dataengine.cp311-win_amd64.pyd +0 -0
- mindspore/_c_expression.cp311-win_amd64.pyd +0 -0
- mindspore/_c_mindrecord.cp311-win_amd64.pyd +0 -0
- mindspore/_check_jit_forbidden_api.py +3 -0
- mindspore/_checkparam.py +3 -33
- mindspore/_deprecated/__init__.py +17 -0
- mindspore/_deprecated/jit.py +198 -0
- mindspore/_extends/builtin_operations.py +1 -1
- mindspore/_extends/parse/__init__.py +6 -7
- mindspore/_extends/parse/compile_config.py +19 -0
- mindspore/_extends/parse/deprecated/deprecated_tensor_method.py +22 -3
- mindspore/_extends/parse/jit_fallback_modules/__init__.py +0 -0
- mindspore/_extends/parse/jit_fallback_modules/check_utils.py +123 -0
- mindspore/_extends/parse/jit_fallback_modules/third_party_modules.py +50 -0
- mindspore/_extends/parse/parser.py +24 -193
- mindspore/_extends/parse/resources.py +1 -5
- mindspore/_extends/parse/standard_method.py +97 -74
- mindspore/_extends/pijit/__init__.py +2 -2
- mindspore/_extends/pijit/pijit_func_white_list.py +16 -11
- mindspore/_extends/pijit/tensor_func_list.py +27 -0
- mindspore/_extends/utils.py +1 -1
- mindspore/amp.py +4 -4
- mindspore/atlprov.dll +0 -0
- mindspore/avcodec-59.dll +0 -0
- mindspore/avdevice-59.dll +0 -0
- mindspore/avfilter-8.dll +0 -0
- mindspore/avformat-59.dll +0 -0
- mindspore/avutil-57.dll +0 -0
- mindspore/boost/__init__.py +2 -2
- mindspore/boost/base.py +3 -7
- mindspore/boost/boost_cell_wrapper.py +2 -2
- mindspore/c1.dll +0 -0
- mindspore/c1xx.dll +0 -0
- mindspore/c2.dll +0 -0
- mindspore/common/__init__.py +4 -3
- mindspore/common/_grad_function.py +56 -0
- mindspore/common/_pijit_context.py +14 -5
- mindspore/common/_register_for_tensor.py +1 -1
- mindspore/common/_stub_tensor.py +5 -10
- mindspore/common/_tensor_cpp_method.py +1 -1
- mindspore/common/_tensor_docs.py +1915 -3287
- mindspore/common/api.py +341 -354
- mindspore/common/auto_dynamic_shape.py +41 -44
- mindspore/common/dtype.py +5 -2
- mindspore/common/dump.py +7 -5
- mindspore/common/file_system.py +3 -0
- mindspore/common/hook_handle.py +5 -3
- mindspore/common/initializer.py +10 -6
- mindspore/common/jit_begin_end.py +94 -0
- mindspore/common/jit_config.py +6 -1
- mindspore/common/jit_context.py +76 -0
- mindspore/common/jit_trace.py +378 -0
- mindspore/common/lazy_inline.py +2 -2
- mindspore/common/mutable.py +5 -4
- mindspore/common/parameter.py +106 -39
- mindspore/common/seed.py +2 -2
- mindspore/common/sparse_tensor.py +23 -17
- mindspore/common/tensor.py +297 -714
- mindspore/communication/__init__.py +7 -5
- mindspore/communication/_comm_helper.py +47 -2
- mindspore/communication/comm_func.py +70 -53
- mindspore/communication/management.py +83 -17
- mindspore/context.py +214 -560
- mindspore/dataset/__init__.py +44 -20
- mindspore/dataset/audio/__init__.py +2 -8
- mindspore/dataset/audio/transforms.py +3 -17
- mindspore/dataset/core/config.py +3 -3
- mindspore/dataset/engine/cache_client.py +1 -1
- mindspore/dataset/engine/datasets.py +102 -120
- mindspore/dataset/engine/datasets_audio.py +22 -22
- mindspore/dataset/engine/datasets_standard_format.py +43 -24
- mindspore/dataset/engine/datasets_text.py +78 -85
- mindspore/dataset/engine/datasets_user_defined.py +108 -76
- mindspore/dataset/engine/datasets_vision.py +111 -108
- mindspore/dataset/engine/iterators.py +5 -3
- mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +1 -1
- mindspore/dataset/engine/samplers.py +279 -57
- mindspore/dataset/engine/serializer_deserializer.py +2 -1
- mindspore/dataset/engine/validators.py +10 -0
- mindspore/dataset/text/__init__.py +7 -6
- mindspore/dataset/text/transforms.py +6 -5
- mindspore/dataset/text/utils.py +3 -3
- mindspore/dataset/transforms/__init__.py +0 -9
- mindspore/dataset/transforms/transforms.py +3 -3
- mindspore/dataset/utils/browse_dataset.py +1 -1
- mindspore/dataset/vision/__init__.py +2 -9
- mindspore/dataset/vision/transforms.py +202 -158
- mindspore/dataset/vision/utils.py +7 -5
- mindspore/device_context/ascend/op_debug.py +60 -1
- mindspore/device_context/ascend/op_tuning.py +0 -4
- mindspore/device_manager.py +39 -3
- mindspore/dnnl.dll +0 -0
- mindspore/dpcmi.dll +0 -0
- mindspore/experimental/es/embedding_service.py +35 -27
- mindspore/experimental/map_parameter.py +4 -4
- mindspore/experimental/optim/adadelta.py +22 -26
- mindspore/experimental/optim/adagrad.py +4 -4
- mindspore/experimental/optim/adam.py +4 -0
- mindspore/experimental/optim/adamax.py +4 -4
- mindspore/experimental/optim/adamw.py +4 -0
- mindspore/experimental/optim/asgd.py +1 -1
- mindspore/experimental/optim/lr_scheduler.py +40 -22
- mindspore/experimental/optim/radam.py +5 -5
- mindspore/experimental/optim/rprop.py +1 -1
- mindspore/experimental/optim/sgd.py +1 -1
- mindspore/hal/contiguous_tensors_handle.py +6 -10
- mindspore/hal/device.py +55 -81
- mindspore/hal/event.py +38 -55
- mindspore/hal/memory.py +93 -144
- mindspore/hal/stream.py +81 -125
- mindspore/include/dataset/constants.h +7 -4
- mindspore/include/dataset/execute.h +2 -2
- mindspore/jpeg62.dll +0 -0
- mindspore/log.py +40 -2
- mindspore/mindrecord/__init__.py +20 -7
- mindspore/mindspore_backend_common.dll +0 -0
- mindspore/mindspore_backend_manager.dll +0 -0
- mindspore/mindspore_common.dll +0 -0
- mindspore/mindspore_core.dll +0 -0
- mindspore/mindspore_dump.dll +0 -0
- mindspore/mindspore_frontend.dll +0 -0
- mindspore/mindspore_glog.dll +0 -0
- mindspore/mindspore_memory_pool.dll +0 -0
- mindspore/mindspore_ms_backend.dll +0 -0
- mindspore/mindspore_ops.dll +0 -0
- mindspore/{mindspore_backend.dll → mindspore_ops_host.dll} +0 -0
- mindspore/mindspore_ops_kernel_common.dll +0 -0
- mindspore/mindspore_profiler.dll +0 -0
- mindspore/mindspore_pyboost.dll +0 -0
- mindspore/mindspore_pynative.dll +0 -0
- mindspore/mindspore_res_manager.dll +0 -0
- mindspore/mindspore_runtime_pipeline.dll +0 -0
- mindspore/mint/__init__.py +131 -700
- mindspore/mint/distributed/__init__.py +5 -1
- mindspore/mint/distributed/distributed.py +194 -109
- mindspore/mint/linalg/__init__.py +2 -0
- mindspore/mint/nn/__init__.py +280 -18
- mindspore/mint/nn/functional.py +282 -64
- mindspore/mint/nn/layer/__init__.py +4 -0
- mindspore/mint/nn/layer/_functions.py +7 -3
- mindspore/mint/nn/layer/activation.py +120 -13
- mindspore/mint/nn/layer/conv.py +218 -24
- mindspore/mint/nn/layer/normalization.py +15 -16
- mindspore/mint/nn/layer/padding.py +1 -1
- mindspore/mint/nn/layer/pooling.py +66 -1
- mindspore/mint/optim/__init__.py +2 -1
- mindspore/mint/optim/sgd.py +171 -0
- mindspore/msobj140.dll +0 -0
- mindspore/mspdb140.dll +0 -0
- mindspore/mspdbcore.dll +0 -0
- mindspore/mspdbst.dll +0 -0
- mindspore/mspft140.dll +0 -0
- mindspore/msvcdis140.dll +0 -0
- mindspore/msvcp140_1.dll +0 -0
- mindspore/msvcp140_2.dll +0 -0
- mindspore/msvcp140_atomic_wait.dll +0 -0
- mindspore/msvcp140_codecvt_ids.dll +0 -0
- mindspore/nn/__init__.py +4 -1
- mindspore/nn/cell.py +1250 -176
- mindspore/nn/layer/activation.py +23 -21
- mindspore/nn/layer/basic.py +22 -16
- mindspore/nn/layer/container.py +1 -1
- mindspore/nn/layer/conv.py +22 -17
- mindspore/nn/layer/embedding.py +9 -8
- mindspore/nn/layer/normalization.py +48 -42
- mindspore/nn/layer/pooling.py +75 -31
- mindspore/nn/layer/transformer.py +11 -10
- mindspore/nn/learning_rate_schedule.py +4 -2
- mindspore/nn/loss/loss.py +27 -19
- mindspore/nn/optim/ada_grad.py +6 -5
- mindspore/nn/optim/adadelta.py +9 -7
- mindspore/nn/optim/adafactor.py +1 -1
- mindspore/nn/optim/adam.py +16 -12
- mindspore/nn/optim/adamax.py +8 -7
- mindspore/nn/optim/adasum.py +5 -5
- mindspore/nn/optim/asgd.py +1 -1
- mindspore/nn/optim/ftrl.py +11 -9
- mindspore/nn/optim/lamb.py +1 -1
- mindspore/nn/optim/lazyadam.py +12 -10
- mindspore/nn/optim/momentum.py +7 -6
- mindspore/nn/optim/optimizer.py +2 -2
- mindspore/nn/optim/proximal_ada_grad.py +12 -10
- mindspore/nn/optim/rmsprop.py +13 -12
- mindspore/nn/optim/rprop.py +9 -7
- mindspore/nn/optim/sgd.py +9 -6
- mindspore/nn/optim/tft_wrapper.py +5 -2
- mindspore/nn/probability/bijector/bijector.py +17 -11
- mindspore/nn/probability/bijector/gumbel_cdf.py +5 -5
- mindspore/nn/probability/bijector/invert.py +2 -2
- mindspore/nn/probability/bijector/scalar_affine.py +3 -3
- mindspore/nn/probability/bijector/softplus.py +3 -2
- mindspore/nn/probability/distribution/beta.py +3 -3
- mindspore/nn/probability/distribution/categorical.py +1 -1
- mindspore/nn/probability/distribution/cauchy.py +4 -2
- mindspore/nn/probability/distribution/exponential.py +6 -7
- mindspore/nn/probability/distribution/gamma.py +2 -2
- mindspore/nn/probability/distribution/gumbel.py +2 -2
- mindspore/nn/probability/distribution/half_normal.py +5 -3
- mindspore/nn/probability/distribution/logistic.py +5 -3
- mindspore/nn/probability/distribution/poisson.py +1 -1
- mindspore/nn/probability/distribution/uniform.py +5 -3
- mindspore/nn/reinforcement/_tensors_queue.py +1 -1
- mindspore/nn/reinforcement/tensor_array.py +1 -1
- mindspore/nn/wrap/__init__.py +6 -6
- mindspore/nn/wrap/cell_wrapper.py +178 -117
- mindspore/nn/wrap/grad_reducer.py +45 -36
- mindspore/nn/wrap/loss_scale.py +3 -3
- mindspore/numpy/array_creations.py +3 -3
- mindspore/numpy/array_ops.py +1 -1
- mindspore/numpy/math_ops.py +4 -4
- mindspore/numpy/utils.py +1 -2
- mindspore/numpy/utils_const.py +1 -2
- mindspore/opencv_core452.dll +0 -0
- mindspore/opencv_imgcodecs452.dll +0 -0
- mindspore/opencv_imgproc452.dll +0 -0
- mindspore/ops/__init__.py +3 -2
- mindspore/ops/_grad_experimental/grad_comm_ops.py +18 -3
- mindspore/ops/_grad_experimental/grad_debug_ops.py +8 -1
- mindspore/ops/_grad_experimental/taylor_rule.py +29 -0
- mindspore/ops/_register_for_op.py +0 -11
- mindspore/{ops_generate → ops/_utils}/arg_dtype_cast.py +123 -4
- mindspore/{ops_generate → ops/_utils}/arg_handler.py +3 -4
- mindspore/ops/_vmap/vmap_array_ops.py +7 -6
- mindspore/ops/_vmap/vmap_grad_nn_ops.py +2 -1
- mindspore/ops/_vmap/vmap_math_ops.py +4 -7
- mindspore/ops/_vmap/vmap_nn_ops.py +9 -8
- mindspore/ops/auto_generate/__init__.py +4 -3
- mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py +102 -49
- mindspore/ops/auto_generate/gen_extend_func.py +281 -135
- mindspore/ops/auto_generate/gen_ops_def.py +2574 -2326
- mindspore/ops/auto_generate/gen_ops_prim.py +8566 -2755
- mindspore/ops/auto_generate/pyboost_inner_prim.py +106 -76
- mindspore/ops/composite/__init__.py +2 -1
- mindspore/ops/composite/base.py +19 -24
- mindspore/ops/composite/math_ops.py +6 -16
- mindspore/ops/composite/multitype_ops/__init__.py +5 -2
- mindspore/ops/composite/multitype_ops/_compile_utils.py +2 -3
- mindspore/ops/composite/multitype_ops/_constexpr_utils.py +1 -2
- mindspore/ops/composite/multitype_ops/add_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/bitwise_and_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/bitwise_or_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/bitwise_xor_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/div_impl.py +6 -4
- mindspore/ops/composite/multitype_ops/equal_impl.py +4 -3
- mindspore/ops/composite/multitype_ops/floordiv_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/getitem_impl.py +3 -2
- mindspore/ops/composite/multitype_ops/greater_equal_impl.py +4 -3
- mindspore/ops/composite/multitype_ops/greater_impl.py +4 -3
- mindspore/ops/composite/multitype_ops/in_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/invert_impl.py +50 -0
- mindspore/ops/composite/multitype_ops/left_shift_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/less_equal_impl.py +4 -3
- mindspore/ops/composite/multitype_ops/less_impl.py +4 -3
- mindspore/ops/composite/multitype_ops/logic_not_impl.py +3 -2
- mindspore/ops/composite/multitype_ops/logical_and_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/logical_or_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/mod_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/mul_impl.py +3 -2
- mindspore/ops/composite/multitype_ops/negative_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/not_equal_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/not_in_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/ones_like_impl.py +18 -0
- mindspore/ops/composite/multitype_ops/pow_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/right_shift_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/setitem_impl.py +2 -1
- mindspore/ops/composite/multitype_ops/sub_impl.py +2 -1
- mindspore/ops/function/__init__.py +28 -2
- mindspore/ops/function/_add_attr_func.py +58 -0
- mindspore/ops/function/array_func.py +1629 -2345
- mindspore/ops/function/clip_func.py +38 -45
- mindspore/ops/function/debug_func.py +36 -44
- mindspore/ops/function/grad/__init__.py +1 -0
- mindspore/ops/function/grad/grad_func.py +104 -71
- mindspore/ops/function/image_func.py +1 -1
- mindspore/ops/function/linalg_func.py +46 -78
- mindspore/ops/function/math_func.py +3035 -3705
- mindspore/ops/function/nn_func.py +676 -241
- mindspore/ops/function/other_func.py +159 -1
- mindspore/ops/function/parameter_func.py +17 -30
- mindspore/ops/function/random_func.py +204 -361
- mindspore/ops/function/reshard_func.py +4 -70
- mindspore/ops/function/sparse_func.py +3 -3
- mindspore/ops/function/sparse_unary_func.py +5 -5
- mindspore/ops/function/spectral_func.py +25 -58
- mindspore/ops/function/vmap_func.py +24 -17
- mindspore/ops/functional.py +6 -4
- mindspore/ops/functional_overload.py +547 -4
- mindspore/ops/op_info_register.py +32 -244
- mindspore/ops/operations/__init__.py +10 -5
- mindspore/ops/operations/_custom_ops_utils.py +247 -0
- mindspore/ops/operations/_grad_ops.py +1 -10
- mindspore/ops/operations/_inner_ops.py +5 -76
- mindspore/ops/operations/_ms_kernel.py +4 -10
- mindspore/ops/operations/_rl_inner_ops.py +1 -1
- mindspore/ops/operations/_scalar_ops.py +3 -2
- mindspore/ops/operations/_sequence_ops.py +1 -1
- mindspore/ops/operations/_tensor_array.py +1 -1
- mindspore/ops/operations/array_ops.py +37 -22
- mindspore/ops/operations/comm_ops.py +150 -107
- mindspore/ops/operations/custom_ops.py +221 -23
- mindspore/ops/operations/debug_ops.py +115 -16
- mindspore/ops/operations/inner_ops.py +1 -1
- mindspore/ops/operations/linalg_ops.py +1 -58
- mindspore/ops/operations/manually_defined/_inner.py +1 -1
- mindspore/ops/operations/manually_defined/ops_def.py +746 -79
- mindspore/ops/operations/math_ops.py +21 -18
- mindspore/ops/operations/nn_ops.py +65 -191
- mindspore/ops/operations/other_ops.py +62 -9
- mindspore/ops/operations/random_ops.py +13 -7
- mindspore/ops/operations/reshard_ops.py +1 -1
- mindspore/ops/operations/sparse_ops.py +2 -2
- mindspore/ops/primitive.py +43 -32
- mindspore/ops/tensor_method.py +232 -13
- mindspore/ops_generate/__init__.py +0 -5
- mindspore/ops_generate/aclnn/__init__.py +0 -0
- mindspore/ops_generate/{aclnn_kernel_register_auto_cc_generator.py → aclnn/aclnn_kernel_register_auto_cc_generator.py} +43 -18
- mindspore/ops_generate/{gen_aclnn_implement.py → aclnn/gen_aclnn_implement.py} +49 -51
- mindspore/ops_generate/api/__init__.py +0 -0
- mindspore/ops_generate/{add_tensor_docs_generator.py → api/add_tensor_docs_generator.py} +9 -7
- mindspore/ops_generate/{cpp_create_prim_instance_helper_generator.py → api/cpp_create_prim_instance_helper_generator.py} +6 -9
- mindspore/ops_generate/{functional_map_cpp_generator.py → api/functional_map_cpp_generator.py} +25 -12
- mindspore/ops_generate/{functional_overload_py_generator.py → api/functional_overload_py_generator.py} +8 -6
- mindspore/ops_generate/{functions_cc_generator.py → api/functions_cc_generator.py} +14 -10
- mindspore/ops_generate/api/gen_api.py +103 -0
- mindspore/ops_generate/{op_api_proto.py → api/op_api_proto.py} +98 -69
- mindspore/ops_generate/{tensor_func_reg_cpp_generator.py → api/tensor_func_reg_cpp_generator.py} +82 -43
- mindspore/ops_generate/common/__init__.py +0 -0
- mindspore/ops_generate/common/gen_constants.py +91 -0
- mindspore/ops_generate/{gen_utils.py → common/gen_utils.py} +72 -19
- mindspore/ops_generate/{op_proto.py → common/op_proto.py} +64 -1
- mindspore/ops_generate/{template.py → common/template.py} +96 -84
- mindspore/ops_generate/gen_ops.py +23 -325
- mindspore/ops_generate/op_def/__init__.py +0 -0
- mindspore/ops_generate/op_def/gen_op_def.py +90 -0
- mindspore/ops_generate/{lite_ops_cpp_generator.py → op_def/lite_ops_cpp_generator.py} +47 -11
- mindspore/ops_generate/{ops_def_cc_generator.py → op_def/ops_def_cc_generator.py} +18 -7
- mindspore/ops_generate/{ops_def_h_generator.py → op_def/ops_def_h_generator.py} +5 -5
- mindspore/ops_generate/{ops_name_h_generator.py → op_def/ops_name_h_generator.py} +30 -15
- mindspore/ops_generate/op_def/ops_primitive_h_generator.py +125 -0
- mindspore/ops_generate/op_def_py/__init__.py +0 -0
- mindspore/ops_generate/op_def_py/gen_op_def_py.py +47 -0
- mindspore/ops_generate/{op_def_py_generator.py → op_def_py/op_def_py_generator.py} +6 -5
- mindspore/ops_generate/{op_prim_py_generator.py → op_def_py/op_prim_py_generator.py} +24 -15
- mindspore/ops_generate/pyboost/__init__.py +0 -0
- mindspore/ops_generate/{auto_grad_impl_cc_generator.py → pyboost/auto_grad_impl_cc_generator.py} +11 -7
- mindspore/ops_generate/{auto_grad_reg_cc_generator.py → pyboost/auto_grad_reg_cc_generator.py} +7 -7
- mindspore/ops_generate/{gen_pyboost_func.py → pyboost/gen_pyboost_func.py} +40 -16
- mindspore/ops_generate/{op_template_parser.py → pyboost/op_template_parser.py} +105 -24
- mindspore/ops_generate/{pyboost_functions_cpp_generator.py → pyboost/pyboost_functions_cpp_generator.py} +55 -18
- mindspore/ops_generate/{pyboost_functions_h_generator.py → pyboost/pyboost_functions_h_generator.py} +42 -10
- mindspore/ops_generate/{pyboost_functions_py_generator.py → pyboost/pyboost_functions_py_generator.py} +6 -6
- mindspore/ops_generate/{pyboost_grad_function_cpp_generator.py → pyboost/pyboost_grad_function_cpp_generator.py} +11 -10
- mindspore/ops_generate/{pyboost_inner_prim_generator.py → pyboost/pyboost_inner_prim_generator.py} +8 -7
- mindspore/ops_generate/{pyboost_native_grad_functions_generator.py → pyboost/pyboost_native_grad_functions_generator.py} +14 -10
- mindspore/ops_generate/{pyboost_op_cpp_code_generator.py → pyboost/pyboost_op_cpp_code_generator.py} +140 -53
- mindspore/ops_generate/{pyboost_overload_functions_cpp_generator.py → pyboost/pyboost_overload_functions_cpp_generator.py} +28 -15
- mindspore/ops_generate/{pyboost_utils.py → pyboost/pyboost_utils.py} +88 -4
- mindspore/ops_generate/resources/__init__.py +0 -0
- mindspore/ops_generate/resources/resource_list.py +30 -0
- mindspore/ops_generate/resources/resource_loader.py +36 -0
- mindspore/ops_generate/resources/resource_manager.py +64 -0
- mindspore/ops_generate/resources/yaml_loader.py +88 -0
- mindspore/ops_generate/tensor_py_cc_generator.py +122 -0
- mindspore/parallel/__init__.py +6 -2
- mindspore/parallel/_auto_parallel_context.py +133 -6
- mindspore/parallel/_cell_wrapper.py +130 -15
- mindspore/parallel/_parallel_serialization.py +95 -4
- mindspore/parallel/_ps_context.py +1 -1
- mindspore/parallel/_recovery_context.py +7 -2
- mindspore/parallel/_tensor.py +142 -18
- mindspore/parallel/_utils.py +198 -25
- mindspore/parallel/algo_parameter_config.py +3 -3
- mindspore/parallel/auto_parallel.py +732 -0
- mindspore/parallel/checkpoint_convert.py +159 -0
- mindspore/parallel/checkpoint_transform.py +656 -37
- mindspore/parallel/cluster/process_entity/_api.py +151 -19
- mindspore/parallel/cluster/run.py +1 -1
- mindspore/parallel/function/__init__.py +24 -0
- mindspore/parallel/function/reshard_func.py +259 -0
- mindspore/parallel/nn/__init__.py +25 -0
- mindspore/parallel/nn/parallel_cell_wrapper.py +263 -0
- mindspore/parallel/nn/parallel_grad_reducer.py +169 -0
- mindspore/parallel/parameter_broadcast.py +24 -13
- mindspore/parallel/shard.py +137 -61
- mindspore/parallel/transform_safetensors.py +287 -95
- mindspore/pgodb140.dll +0 -0
- mindspore/pgort140.dll +0 -0
- mindspore/profiler/__init__.py +9 -5
- mindspore/profiler/analysis/parser/ascend_cann_parser.py +6 -2
- mindspore/profiler/analysis/parser/ms_framework_parser.py +4 -4
- mindspore/profiler/analysis/parser/timeline_assembly_factory/ascend_timeline_assembler.py +7 -4
- mindspore/profiler/analysis/parser/timeline_assembly_factory/trace_view_container.py +22 -0
- mindspore/profiler/analysis/parser/timeline_creator/fwk_timeline_creator.py +3 -3
- mindspore/profiler/analysis/parser/timeline_event/fwk_event.py +241 -86
- mindspore/profiler/analysis/viewer/ascend_communication_viewer.py +41 -2
- mindspore/profiler/analysis/viewer/ascend_kernel_details_viewer.py +33 -35
- mindspore/profiler/analysis/viewer/ascend_memory_viewer.py +7 -0
- mindspore/profiler/analysis/viewer/ascend_op_memory_viewer.py +8 -3
- mindspore/profiler/analysis/viewer/ascend_step_trace_time_viewer.py +141 -30
- mindspore/profiler/analysis/viewer/ms_dataset_viewer.py +5 -6
- mindspore/profiler/common/ascend_msprof_exporter.py +5 -4
- mindspore/profiler/common/constant.py +12 -0
- mindspore/profiler/common/msprof_cmd_tool.py +42 -23
- mindspore/profiler/common/path_manager.py +24 -0
- mindspore/profiler/common/profiler_context.py +26 -2
- mindspore/profiler/common/profiler_meta_data.py +74 -0
- mindspore/profiler/common/profiler_parameters.py +59 -18
- mindspore/profiler/common/profiler_path_manager.py +66 -7
- mindspore/profiler/dynamic_profiler.py +112 -79
- mindspore/profiler/envprofiler.py +26 -1
- mindspore/profiler/experimental_config.py +197 -0
- mindspore/profiler/mstx.py +57 -14
- mindspore/profiler/platform/npu_profiler.py +33 -7
- mindspore/profiler/profiler.py +541 -45
- mindspore/profiler/profiler_action_controller.py +1 -1
- mindspore/profiler/profiler_interface.py +4 -0
- mindspore/profiler/schedule.py +57 -22
- mindspore/rewrite/api/node.py +15 -13
- mindspore/rewrite/api/symbol_tree.py +1 -1
- mindspore/run_check/_check_version.py +25 -14
- mindspore/run_check/run_check.py +1 -1
- mindspore/runtime/__init__.py +2 -2
- mindspore/runtime/executor.py +40 -11
- mindspore/runtime/memory.py +25 -8
- mindspore/safeguard/rewrite_obfuscation.py +12 -9
- mindspore/swresample-4.dll +0 -0
- mindspore/swscale-6.dll +0 -0
- mindspore/tbbmalloc.dll +0 -0
- mindspore/tinyxml2.dll +0 -0
- mindspore/train/__init__.py +8 -8
- mindspore/train/_utils.py +35 -7
- mindspore/train/amp.py +1 -1
- mindspore/train/callback/__init__.py +2 -2
- mindspore/train/callback/_callback.py +2 -16
- mindspore/train/callback/_checkpoint.py +24 -40
- mindspore/train/callback/_cluster_monitor.py +14 -18
- mindspore/train/callback/_flops_collector.py +2 -3
- mindspore/train/callback/_history.py +7 -4
- mindspore/train/callback/_lambda_callback.py +2 -2
- mindspore/train/callback/_landscape.py +0 -3
- mindspore/train/callback/_loss_monitor.py +2 -1
- mindspore/train/callback/_on_request_exit.py +6 -5
- mindspore/train/callback/_reduce_lr_on_plateau.py +11 -6
- mindspore/train/callback/_summary_collector.py +8 -13
- mindspore/train/callback/_time_monitor.py +2 -1
- mindspore/train/callback/{_tft_register.py → _train_fault_tolerance.py} +179 -103
- mindspore/train/data_sink.py +25 -2
- mindspore/train/dataset_helper.py +4 -5
- mindspore/train/loss_scale_manager.py +8 -7
- mindspore/train/metrics/accuracy.py +3 -3
- mindspore/train/metrics/confusion_matrix.py +9 -9
- mindspore/train/metrics/error.py +3 -3
- mindspore/train/metrics/hausdorff_distance.py +4 -4
- mindspore/train/metrics/mean_surface_distance.py +3 -3
- mindspore/train/metrics/metric.py +0 -12
- mindspore/train/metrics/occlusion_sensitivity.py +4 -2
- mindspore/train/metrics/precision.py +8 -6
- mindspore/train/metrics/recall.py +9 -9
- mindspore/train/metrics/root_mean_square_surface_distance.py +2 -2
- mindspore/train/mind_ir_pb2.py +19 -12
- mindspore/train/model.py +176 -103
- mindspore/train/serialization.py +246 -988
- mindspore/train/summary/_summary_adapter.py +2 -2
- mindspore/train/summary/summary_record.py +1 -1
- mindspore/turbojpeg.dll +0 -0
- mindspore/utils/__init__.py +3 -2
- mindspore/utils/dryrun.py +4 -2
- mindspore/utils/hooks.py +81 -0
- mindspore/utils/utils.py +138 -4
- mindspore/vcmeta.dll +0 -0
- mindspore/vcruntime140.dll +0 -0
- mindspore/vcruntime140_1.dll +0 -0
- mindspore/version.py +1 -1
- {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/METADATA +2 -1
- {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/RECORD +483 -438
- mindspore/_install_custom.py +0 -43
- mindspore/common/_register_for_adapter.py +0 -74
- mindspore/ops/auto_generate/gen_arg_dtype_cast.py +0 -252
- mindspore/ops/auto_generate/gen_arg_handler.py +0 -136
- mindspore/ops/operations/_opaque_predicate_registry.py +0 -41
- mindspore/ops_generate/gen_constants.py +0 -190
- mindspore/ops_generate/gen_ops_inner_prim.py +0 -131
- mindspore/ops_generate/ops_primitive_h_generator.py +0 -81
- /mindspore/ops_generate/{base_generator.py → common/base_generator.py} +0 -0
- {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/WHEEL +0 -0
- {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/entry_points.txt +0 -0
- {mindspore-2.5.0.dist-info → mindspore-2.6.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# Copyright 2023 Huawei Technologies Co., Ltd
|
|
1
|
+
# Copyright 2023-2025 Huawei Technologies Co., Ltd
|
|
2
2
|
#
|
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
# you may not use this file except in compliance with the License.
|
|
@@ -13,342 +13,40 @@
|
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
# ============================================================================
|
|
15
15
|
"""
|
|
16
|
-
|
|
16
|
+
Auto generate ops files.
|
|
17
17
|
"""
|
|
18
|
-
import argparse
|
|
19
|
-
import copy
|
|
20
18
|
import logging
|
|
21
|
-
import os
|
|
22
|
-
import shutil
|
|
23
|
-
import pathlib
|
|
24
|
-
from gen_utils import (check_change_and_replace_file, merge_files,
|
|
25
|
-
merge_files_append, safe_load_yaml)
|
|
26
|
-
from op_prim_py_generator import OpPrimPyGenerator
|
|
27
|
-
from op_def_py_generator import OpDefPyGenerator
|
|
28
|
-
from aclnn_kernel_register_auto_cc_generator import AclnnKernelRegisterAutoCcGenerator
|
|
29
|
-
from cpp_create_prim_instance_helper_generator import CppCreatePrimInstanceHelperGenerator
|
|
30
|
-
from ops_def_cc_generator import OpsDefCcGenerator
|
|
31
|
-
from ops_def_h_generator import OpsDefHGenerator
|
|
32
|
-
from ops_primitive_h_generator import OpsPrimitiveHGenerator
|
|
33
|
-
from lite_ops_cpp_generator import LiteOpsCcGenerator, LiteOpsHGenerator
|
|
34
|
-
from ops_name_h_generator import OpsNameHGenerator
|
|
35
|
-
from functional_map_cpp_generator import FunctionalMapCppGenerator
|
|
36
|
-
from add_tensor_docs_generator import AddTensorDocsGenerator
|
|
37
|
-
from functional_overload_py_generator import FunctionalOverloadPyGenerator
|
|
38
19
|
|
|
39
|
-
from
|
|
40
|
-
from
|
|
41
|
-
from tensor_func_reg_cpp_generator import TensorFuncRegCppGenerator
|
|
42
|
-
from gen_pyboost_func import gen_pyboost_code
|
|
20
|
+
from resources.resource_manager import prepare_resources
|
|
21
|
+
from common import gen_utils
|
|
43
22
|
|
|
44
|
-
|
|
23
|
+
from op_def.gen_op_def import generate_ops_def_files
|
|
24
|
+
from op_def_py.gen_op_def_py import generate_ops_py_files
|
|
25
|
+
from api.gen_api import generate_api_files
|
|
26
|
+
from aclnn.aclnn_kernel_register_auto_cc_generator import generate_aclnn_reg_file
|
|
27
|
+
from pyboost.gen_pyboost_func import gen_pyboost_code
|
|
45
28
|
|
|
46
29
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
30
|
+
module_generators = [
|
|
31
|
+
generate_ops_py_files, # generate ops python files
|
|
32
|
+
generate_ops_def_files, # generate ops definition files
|
|
33
|
+
gen_pyboost_code, # generate pyboost code
|
|
34
|
+
generate_aclnn_reg_file, # generate aclnn kernelmod register
|
|
35
|
+
generate_api_files # generate api definition files
|
|
36
|
+
]
|
|
50
37
|
|
|
51
38
|
|
|
52
|
-
def
|
|
53
|
-
|
|
54
|
-
generator.generate(work_path, os_protos, doc_dict, file_pre)
|
|
39
|
+
def main():
|
|
40
|
+
resource_mgr = prepare_resources()
|
|
55
41
|
|
|
42
|
+
for generator in module_generators:
|
|
43
|
+
generator(resource_mgr)
|
|
56
44
|
|
|
57
|
-
|
|
58
|
-
"""
|
|
59
|
-
Generate ops python file from yaml.
|
|
60
|
-
"""
|
|
61
|
-
generate_ops_prim_file(work_path, op_protos, doc_dict, file_pre)
|
|
62
|
-
generate_ops_def_file(work_path, op_protos, doc_dict, file_pre)
|
|
63
|
-
shutil.copy(os.path.join(work_path, K.PY_OPS_GEN_PATH, 'ops_auto_generate_init.txt'),
|
|
64
|
-
os.path.join(work_path, K.PY_AUTO_GEN_PATH, "__init__.py"))
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
def call_ops_def_cc_generator(work_path, op_protos):
|
|
68
|
-
generator = OpsDefCcGenerator()
|
|
69
|
-
generator.generate(work_path, op_protos)
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def call_ops_def_h_generator(work_path, op_protos):
|
|
73
|
-
generator = OpsDefHGenerator()
|
|
74
|
-
generator.generate(work_path, op_protos)
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
def call_ops_primitive_h_generator(work_path, op_protos):
|
|
78
|
-
generator = OpsPrimitiveHGenerator()
|
|
79
|
-
generator.generate(work_path, op_protos)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
def call_lite_ops_h_generator(work_path, op_protos):
|
|
83
|
-
h_generator = LiteOpsHGenerator()
|
|
84
|
-
h_generator.generate(work_path, op_protos)
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
def call_lite_ops_cc_generator(work_path, op_protos):
|
|
88
|
-
generator = LiteOpsCcGenerator()
|
|
89
|
-
generator.generate(work_path, op_protos)
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
def call_ops_name_h_generator(work_path, op_protos):
|
|
93
|
-
h_generator = OpsNameHGenerator()
|
|
94
|
-
h_generator.generate(work_path, op_protos)
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
def generate_ops_cc_files(work_path, op_protos, op_protos_with_deprecated):
|
|
98
|
-
"""
|
|
99
|
-
Generate ops c++ file from yaml.
|
|
100
|
-
"""
|
|
101
|
-
call_ops_def_cc_generator(work_path, op_protos_with_deprecated)
|
|
102
|
-
call_ops_def_h_generator(work_path, op_protos_with_deprecated)
|
|
103
|
-
call_ops_primitive_h_generator(work_path, op_protos)
|
|
104
|
-
call_lite_ops_h_generator(work_path, op_protos)
|
|
105
|
-
call_lite_ops_cc_generator(work_path, op_protos)
|
|
106
|
-
call_ops_name_h_generator(work_path, op_protos)
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
def get_tensor_op_protos_with_deprecated(func_protos, op_protos):
|
|
110
|
-
"""
|
|
111
|
-
Get op_protos with deprecated op_protos from func_protos.
|
|
112
|
-
"""
|
|
113
|
-
tensor_op_protos = copy.deepcopy(op_protos)
|
|
114
|
-
for _, item in func_protos.items():
|
|
115
|
-
for func_proto in item:
|
|
116
|
-
op_name = func_proto.op_proto.op_name
|
|
117
|
-
if "deprecated" in func_proto.op_proto.op_name:
|
|
118
|
-
func_proto.op_proto.op_class.name = ''.join(word.capitalize() for word in op_name.split('_'))
|
|
119
|
-
if func_proto.op_proto.op_name[-1] == '_':
|
|
120
|
-
func_proto.op_proto.op_class.name += '_'
|
|
121
|
-
tensor_op_protos.append(func_proto.op_proto)
|
|
122
|
-
return tensor_op_protos
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
def generate_create_instance_helper_file(work_path, op_protos_with_deprecated):
|
|
126
|
-
"""
|
|
127
|
-
Generate C++ helper file from yaml.
|
|
128
|
-
"""
|
|
129
|
-
generator = CppCreatePrimInstanceHelperGenerator()
|
|
130
|
-
generator.generate(work_path, op_protos_with_deprecated)
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def generate_aclnn_reg_file(work_path, op_protos):
|
|
134
|
-
"""
|
|
135
|
-
Generate nnacl kernelmod register
|
|
136
|
-
"""
|
|
137
|
-
generator = AclnnKernelRegisterAutoCcGenerator()
|
|
138
|
-
generator.generate(work_path, op_protos)
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
def generate_arg_handler_files(work_path):
|
|
142
|
-
"""
|
|
143
|
-
Generate arg handler files.
|
|
144
|
-
"""
|
|
145
|
-
dst_dir = os.path.join(work_path, K.PY_AUTO_GEN_PATH)
|
|
146
|
-
src_arg_handler_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'arg_handler.py')
|
|
147
|
-
dst_arg_handler_path = os.path.join(dst_dir, 'gen_arg_handler.py')
|
|
148
|
-
tmp_dst_arg_handler_path = os.path.join(dst_dir, 'tmp_gen_arg_handler.py')
|
|
149
|
-
if not os.path.exists(dst_dir):
|
|
150
|
-
os.makedirs(dst_dir, mode=0o700)
|
|
151
|
-
shutil.copy(src_arg_handler_path, tmp_dst_arg_handler_path)
|
|
152
|
-
check_change_and_replace_file(dst_arg_handler_path, tmp_dst_arg_handler_path)
|
|
153
|
-
|
|
154
|
-
src_arg_dtype_cast_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'arg_dtype_cast.py')
|
|
155
|
-
dst_arg_dtype_cast_path = os.path.join(dst_dir, 'gen_arg_dtype_cast.py')
|
|
156
|
-
tmp_arg_dtype_cast_path = os.path.join(dst_dir, 'tmp_arg_dtype_cast.py')
|
|
157
|
-
shutil.copy(src_arg_dtype_cast_path, tmp_arg_dtype_cast_path)
|
|
158
|
-
check_change_and_replace_file(dst_arg_dtype_cast_path, tmp_arg_dtype_cast_path)
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
def gen_tensor_func_code(work_path, op_protos, func_protos, alias_api_mapping):
|
|
162
|
-
generator = TensorFuncRegCppGenerator()
|
|
163
|
-
generator.generate(work_path, op_protos, func_protos, alias_api_mapping)
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
def gen_functional_map_code(work_path, tensor_method_protos, mint_func_protos, alias_api_mapping):
|
|
167
|
-
generator = FunctionalMapCppGenerator()
|
|
168
|
-
generator.generate(work_path, tensor_method_protos, mint_func_protos, alias_api_mapping)
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
def gen_tensor_docs_code(work_path, tensor_docs_data):
|
|
172
|
-
generator = AddTensorDocsGenerator()
|
|
173
|
-
generator.generate(work_path, tensor_docs_data)
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
def gen_functional_overload_py(work_path, mint_func_protos, function_doc_data, alias_api_mapping):
|
|
177
|
-
generator = FunctionalOverloadPyGenerator()
|
|
178
|
-
generator.generate(work_path, mint_func_protos, function_doc_data, alias_api_mapping)
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
def main(args):
|
|
182
|
-
current_path = os.path.dirname(os.path.realpath(__file__))
|
|
183
|
-
work_path = os.path.join(current_path, '../../../../')
|
|
184
|
-
|
|
185
|
-
if args.clear_auto_gen:
|
|
186
|
-
delete_auto_gen_files(work_path)
|
|
187
|
-
|
|
188
|
-
# merge ops yaml
|
|
189
|
-
(doc_yaml_path, ops_yaml_path, deprecated_ops_yaml_path, ops_api_yaml_path,
|
|
190
|
-
tensor_method_doc_yaml_path, mint_func_doc_yaml_path) = merge_ops_yaml(work_path)
|
|
191
|
-
|
|
192
|
-
# make auto_generate dir
|
|
193
|
-
cc_path = os.path.join(work_path, K.MS_OP_DEF_AUTO_GENERATE_PATH)
|
|
194
|
-
pathlib.Path(cc_path).mkdir(parents=True, exist_ok=True)
|
|
195
|
-
|
|
196
|
-
# generate arg_handler files
|
|
197
|
-
generate_arg_handler_files(work_path)
|
|
198
|
-
|
|
199
|
-
# read ops definition str and tensor method doc str
|
|
200
|
-
ops_yaml_dict = safe_load_yaml(ops_yaml_path)
|
|
201
|
-
doc_yaml_dict = safe_load_yaml(doc_yaml_path)
|
|
202
|
-
deprecated_ops_yaml_dict = safe_load_yaml(deprecated_ops_yaml_path)
|
|
203
|
-
ops_api_yaml_dict = safe_load_yaml(ops_api_yaml_path)
|
|
204
|
-
tensor_method_doc_yaml_dict = safe_load_yaml(tensor_method_doc_yaml_path)
|
|
205
|
-
mint_function_doc_yaml_dict = safe_load_yaml(mint_func_doc_yaml_path)
|
|
206
|
-
|
|
207
|
-
op_protos = load_op_protos_from_ops_yaml(ops_yaml_dict)
|
|
208
|
-
deprecated_op_protos = load_deprecated_op_protos_from_ops_yaml(deprecated_ops_yaml_dict)
|
|
209
|
-
tensor_method_protos, mint_func_protos, alias_api_mapping \
|
|
210
|
-
= load_api_protos_from_yaml(ops_api_yaml_dict, op_protos, deprecated_op_protos)
|
|
211
|
-
# for generate tensor method deprecated in graph mode
|
|
212
|
-
op_protos_with_deprecated = get_tensor_op_protos_with_deprecated(tensor_method_protos, op_protos)
|
|
213
|
-
|
|
214
|
-
# generate ops python files
|
|
215
|
-
generate_ops_py_files(work_path, op_protos, doc_yaml_dict, "gen")
|
|
216
|
-
# generate ops c++ files
|
|
217
|
-
generate_ops_cc_files(work_path, op_protos, op_protos_with_deprecated)
|
|
218
|
-
# generate create prim instance helper file
|
|
219
|
-
generate_create_instance_helper_file(work_path, op_protos_with_deprecated)
|
|
220
|
-
# generate pyboost code
|
|
221
|
-
gen_pyboost_code(work_path, op_protos, doc_yaml_dict, tensor_method_protos, mint_func_protos, alias_api_mapping)
|
|
222
|
-
# generate aclnn kernelmod register
|
|
223
|
-
generate_aclnn_reg_file(work_path, op_protos)
|
|
224
|
-
# generate tensor_py func code
|
|
225
|
-
gen_tensor_func_code(work_path, op_protos, tensor_method_protos, alias_api_mapping)
|
|
226
|
-
# generate functional map code
|
|
227
|
-
gen_functional_map_code(work_path, tensor_method_protos, mint_func_protos, alias_api_mapping)
|
|
228
|
-
# generate _tensor_docs.py that attaches docs to tensor func APIs when import mindspore
|
|
229
|
-
gen_tensor_docs_code(work_path, tensor_method_doc_yaml_dict)
|
|
230
|
-
# generate functional_overload.py which init pybind mint APIs from cpp
|
|
231
|
-
gen_functional_overload_py(work_path, mint_func_protos, mint_function_doc_yaml_dict, alias_api_mapping)
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
def delete_auto_gen_files(work_path):
|
|
235
|
-
"""
|
|
236
|
-
Deletes auto-generated files and folders.
|
|
237
|
-
"""
|
|
238
|
-
auto_gen_code_file = get_auto_gen_path_from_gitignore(work_path)
|
|
239
|
-
|
|
240
|
-
for name in auto_gen_code_file:
|
|
241
|
-
# Recursively delete all single-level folder names
|
|
242
|
-
if name.rstrip('/').count('/') == 0:
|
|
243
|
-
for dir_path, dir_names, _ in os.walk(work_path, topdown=False):
|
|
244
|
-
for dirname in dir_names:
|
|
245
|
-
if dirname == name.rstrip('/'):
|
|
246
|
-
folder_path = os.path.join(dir_path, dirname)
|
|
247
|
-
logging.info("Recursively deleting folder: %s", folder_path)
|
|
248
|
-
shutil.rmtree(folder_path)
|
|
249
|
-
continue
|
|
250
|
-
|
|
251
|
-
# Delete all individual files or folders
|
|
252
|
-
tmp_path = os.path.join(work_path, name)
|
|
253
|
-
if os.path.exists(tmp_path):
|
|
254
|
-
if os.path.isdir(tmp_path):
|
|
255
|
-
logging.info("Deleting folder: %s", tmp_path)
|
|
256
|
-
shutil.rmtree(tmp_path)
|
|
257
|
-
elif os.path.isfile(tmp_path):
|
|
258
|
-
logging.info("Deleting file: %s", tmp_path)
|
|
259
|
-
os.remove(tmp_path)
|
|
260
|
-
else:
|
|
261
|
-
logging.info("The path is not exist: %s", tmp_path)
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
def get_auto_gen_path_from_gitignore(work_path):
|
|
265
|
-
"""
|
|
266
|
-
Extracts a list of auto-gen file and folder paths from the "# auto gen code files" section in the .gitignore file.
|
|
267
|
-
"""
|
|
268
|
-
file_path = os.path.join(work_path, ".gitignore")
|
|
269
|
-
auto_gen_code_file_started = False
|
|
270
|
-
auto_gen_code_file = []
|
|
271
|
-
with open(file_path, 'r') as f:
|
|
272
|
-
for line in f.readlines():
|
|
273
|
-
if line.strip() == "# auto gen code files":
|
|
274
|
-
auto_gen_code_file_started = True
|
|
275
|
-
continue
|
|
276
|
-
if auto_gen_code_file_started:
|
|
277
|
-
if line.strip() and not line.strip().startswith("#"):
|
|
278
|
-
auto_gen_code_file.append(line.strip())
|
|
279
|
-
else:
|
|
280
|
-
break
|
|
281
|
-
return auto_gen_code_file
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
def load_op_protos_from_ops_yaml(ops_yaml_data):
|
|
285
|
-
op_protos = []
|
|
286
|
-
for operator_name, operator_data in ops_yaml_data.items():
|
|
287
|
-
op_proto = OpProto.load_from_yaml(operator_name, operator_data)
|
|
288
|
-
op_protos.append(op_proto)
|
|
289
|
-
return op_protos
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
def load_deprecated_op_protos_from_ops_yaml(ops_yaml_data):
|
|
293
|
-
op_protos = []
|
|
294
|
-
for operator_name, operator_data in ops_yaml_data.items():
|
|
295
|
-
op_proto = OpProto.load_from_yaml(operator_name, operator_data)
|
|
296
|
-
op_proto.op_name = 'deprecated_' + operator_name
|
|
297
|
-
op_protos.append(op_proto)
|
|
298
|
-
return op_protos
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
def merge_ops_yaml(work_path):
|
|
302
|
-
"""
|
|
303
|
-
Merges operator YAML files scattered in different directories into a single file.
|
|
304
|
-
|
|
305
|
-
Args:
|
|
306
|
-
work_path (str): The path to the working directory.
|
|
307
|
-
|
|
308
|
-
Returns:
|
|
309
|
-
tuple: Paths to the merged documentation and operators YAML files.
|
|
310
|
-
"""
|
|
311
|
-
ops_yaml_dir_path = os.path.join(work_path, K.MS_OP_DEF_YAML_PATH)
|
|
312
|
-
ops_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'ops.yaml')
|
|
313
|
-
infer_ops_yaml_dir_path = os.path.join(ops_yaml_dir_path, "infer")
|
|
314
|
-
merge_files(ops_yaml_dir_path, ops_yaml_path, '*op.yaml')
|
|
315
|
-
merge_files_append(infer_ops_yaml_dir_path, ops_yaml_path, '*op.yaml')
|
|
316
|
-
|
|
317
|
-
doc_yaml_dir_path = os.path.join(ops_yaml_dir_path, "doc")
|
|
318
|
-
doc_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'ops_doc.yaml')
|
|
319
|
-
merge_files(doc_yaml_dir_path, doc_yaml_path, '*doc.yaml')
|
|
320
|
-
|
|
321
|
-
ops_api_yaml_dir_path = os.path.join(work_path, K.MS_OP_API_YAML_PATH)
|
|
322
|
-
ops_api_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'api_def.yaml')
|
|
323
|
-
merge_files(ops_api_yaml_dir_path, ops_api_yaml_path, '*.yaml')
|
|
324
|
-
|
|
325
|
-
deprecated_ops_yaml_dir_path = os.path.join(work_path, K.MS_OP_DEPRECATED_DEF_YAML_PATH)
|
|
326
|
-
deprecated_ops_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'deprecated_ops.yaml')
|
|
327
|
-
merge_files(deprecated_ops_yaml_dir_path, deprecated_ops_yaml_path, '*_method.yaml')
|
|
328
|
-
|
|
329
|
-
tensor_method_doc_yaml_dir_path = os.path.join(work_path, K.MS_TENSOR_METHOD_DOC_YAML_PATH)
|
|
330
|
-
tensor_method_doc_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'tensor_method_doc.yaml')
|
|
331
|
-
merge_files(tensor_method_doc_yaml_dir_path, tensor_method_doc_yaml_path, '*doc.yaml')
|
|
332
|
-
|
|
333
|
-
mint_func_doc_yaml_dir_path = os.path.join(work_path, K.MS_MINT_FUNC_DOC_YAML_PATH)
|
|
334
|
-
mint_func_doc_yaml_path = os.path.join(work_path, K.PY_OPS_GEN_PATH, 'mint_func_doc.yaml')
|
|
335
|
-
merge_files(mint_func_doc_yaml_dir_path, mint_func_doc_yaml_path, '*doc.yaml')
|
|
336
|
-
|
|
337
|
-
return (doc_yaml_path, ops_yaml_path, deprecated_ops_yaml_path,
|
|
338
|
-
ops_api_yaml_path, tensor_method_doc_yaml_path, mint_func_doc_yaml_path)
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
def parse_args():
|
|
342
|
-
parser = argparse.ArgumentParser()
|
|
343
|
-
parser.add_argument('--clear_auto_gen', default=False, help='clear all auto gen files')
|
|
344
|
-
return parser.parse_args()
|
|
345
|
-
|
|
45
|
+
gen_utils.clear_obsolete_auto_gen_files()
|
|
346
46
|
|
|
347
47
|
if __name__ == "__main__":
|
|
348
48
|
try:
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
# pylint: disable=broad-except
|
|
352
|
-
except Exception as e:
|
|
49
|
+
main()
|
|
50
|
+
except Exception as e: # pylint: disable=broad-except
|
|
353
51
|
logging.critical("Auto generate failed, err info: %s", e)
|
|
354
52
|
raise e
|
|
File without changes
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# Copyright 2025 Huawei Technologies Co., Ltd
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
# ============================================================================
|
|
15
|
+
"""
|
|
16
|
+
Generate operator definitions from ops.yaml
|
|
17
|
+
"""
|
|
18
|
+
import copy
|
|
19
|
+
|
|
20
|
+
from common import gen_constants as K
|
|
21
|
+
from resources.resource_list import ResourceType
|
|
22
|
+
|
|
23
|
+
from .ops_def_cc_generator import OpsDefCcGenerator
|
|
24
|
+
from .ops_def_h_generator import OpsDefHGenerator
|
|
25
|
+
from .ops_name_h_generator import OpsNameHGenerator
|
|
26
|
+
from .ops_primitive_h_generator import OpsPrimitiveHGenerator
|
|
27
|
+
from .lite_ops_cpp_generator import LiteOpsCcGenerator, LiteOpsHGenerator
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def call_ops_def_cc_generator(work_path, op_protos):
|
|
31
|
+
generator = OpsDefCcGenerator()
|
|
32
|
+
generator.generate(work_path, op_protos)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def call_ops_def_h_generator(work_path, op_protos):
|
|
36
|
+
generator = OpsDefHGenerator()
|
|
37
|
+
generator.generate(work_path, op_protos)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def call_ops_primitive_h_generator(work_path, op_protos):
|
|
41
|
+
generator = OpsPrimitiveHGenerator()
|
|
42
|
+
generator.generate(work_path, op_protos)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def call_lite_ops_h_generator(work_path, op_protos):
|
|
46
|
+
h_generator = LiteOpsHGenerator()
|
|
47
|
+
h_generator.generate(work_path, op_protos)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def call_lite_ops_cc_generator(work_path, op_protos):
|
|
51
|
+
generator = LiteOpsCcGenerator()
|
|
52
|
+
generator.generate(work_path, op_protos)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def call_ops_name_h_generator(work_path, op_protos):
|
|
56
|
+
h_generator = OpsNameHGenerator()
|
|
57
|
+
h_generator.generate(work_path, op_protos)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_tensor_op_protos_with_deprecated(func_protos, op_protos):
|
|
61
|
+
"""
|
|
62
|
+
Get op_protos with deprecated op_protos from func_protos.
|
|
63
|
+
"""
|
|
64
|
+
tensor_op_protos = copy.deepcopy(op_protos)
|
|
65
|
+
for _, item in func_protos.items():
|
|
66
|
+
for func_proto in item:
|
|
67
|
+
op_name = func_proto.op_proto.op_name
|
|
68
|
+
if "deprecated" in func_proto.op_proto.op_name:
|
|
69
|
+
func_proto.op_proto.op_class.name = ''.join(word.capitalize() for word in op_name.split('_'))
|
|
70
|
+
if func_proto.op_proto.op_name[-1] == '_':
|
|
71
|
+
func_proto.op_proto.op_class.name += '_'
|
|
72
|
+
tensor_op_protos.append(func_proto.op_proto)
|
|
73
|
+
return tensor_op_protos
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def generate_ops_def_files(resource_mgr):
|
|
77
|
+
"""
|
|
78
|
+
Generate ops c++ file from yaml.
|
|
79
|
+
"""
|
|
80
|
+
work_path = K.WORK_DIR
|
|
81
|
+
op_protos = resource_mgr.get_resource(ResourceType.OP_PROTO)
|
|
82
|
+
tensor_method_protos = resource_mgr.get_resource(ResourceType.TENSOR_METHOD_PROTOS)
|
|
83
|
+
# for generate tensor method deprecated in graph mode
|
|
84
|
+
op_protos_with_deprecated = get_tensor_op_protos_with_deprecated(tensor_method_protos, op_protos)
|
|
85
|
+
call_ops_def_cc_generator(work_path, op_protos_with_deprecated)
|
|
86
|
+
call_ops_def_h_generator(work_path, op_protos_with_deprecated)
|
|
87
|
+
call_ops_primitive_h_generator(work_path, op_protos)
|
|
88
|
+
call_lite_ops_h_generator(work_path, op_protos)
|
|
89
|
+
call_lite_ops_cc_generator(work_path, op_protos)
|
|
90
|
+
call_ops_name_h_generator(work_path, op_protos)
|
|
@@ -18,14 +18,44 @@ Generates C++ header and source files for lite operations based on YAML configur
|
|
|
18
18
|
|
|
19
19
|
import os
|
|
20
20
|
|
|
21
|
-
import gen_constants as K
|
|
22
|
-
import gen_utils
|
|
23
|
-
import
|
|
21
|
+
import common.gen_constants as K
|
|
22
|
+
import common.gen_utils as gen_utils
|
|
23
|
+
import common.template as template
|
|
24
|
+
from common.base_generator import BaseGenerator
|
|
25
|
+
from pyboost import pyboost_utils
|
|
24
26
|
|
|
25
|
-
# refactored
|
|
26
|
-
import template
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
LITE_OPS_H = """
|
|
29
|
+
#ifndef MINDSPORE_CORE_OPS_GEN_LITE_OPS_H_
|
|
30
|
+
#define MINDSPORE_CORE_OPS_GEN_LITE_OPS_H_
|
|
31
|
+
|
|
32
|
+
#include <vector>
|
|
33
|
+
#include "ops/base_operator.h"
|
|
34
|
+
|
|
35
|
+
namespace mindspore::ops {
|
|
36
|
+
$ops_namespace_body
|
|
37
|
+
|
|
38
|
+
} // namespace mindspore::ops
|
|
39
|
+
#endif // MINDSPORE_CORE_OPS_GEN_LITE_OPS_H_
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
INC_OPS_HEAD = """
|
|
43
|
+
#include "$auto_gen_path/gen_ops_name_${ch}.h"
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
LITE_OPS_CC = """
|
|
47
|
+
#include "$auto_gen_path/gen_lite_ops.h"
|
|
48
|
+
${inc_ops_head_str}
|
|
49
|
+
#include "mindapi/helper.h"
|
|
50
|
+
#include "ops/primitive_c.h"
|
|
51
|
+
#include "ops/base_operator.h"
|
|
52
|
+
#include "abstract/abstract_value.h"
|
|
53
|
+
|
|
54
|
+
namespace mindspore::ops {
|
|
55
|
+
$ops_namespace_body
|
|
56
|
+
|
|
57
|
+
} // namespace mindspore::ops
|
|
58
|
+
"""
|
|
29
59
|
|
|
30
60
|
|
|
31
61
|
class LiteOpsHGenerator(BaseGenerator):
|
|
@@ -37,7 +67,7 @@ class LiteOpsHGenerator(BaseGenerator):
|
|
|
37
67
|
"""
|
|
38
68
|
Initializes the generator with the necessary templates for generating C++ header files.
|
|
39
69
|
"""
|
|
40
|
-
self.lite_ops_h_template = template.Template(
|
|
70
|
+
self.lite_ops_h_template = template.Template(LITE_OPS_H)
|
|
41
71
|
self.lite_ops_class_template = template.op_cc_template
|
|
42
72
|
self.arg_prim_init_template = template.Template("\n"
|
|
43
73
|
" void set_${arg_name}(const ${dtype} &${arg_name});\n"
|
|
@@ -89,9 +119,11 @@ class LiteOpsCcGenerator(BaseGenerator):
|
|
|
89
119
|
"""
|
|
90
120
|
Initializes the generator with the necessary templates for generating C++ source files.
|
|
91
121
|
"""
|
|
92
|
-
self.
|
|
122
|
+
self.inc_ops_head_templat = template.Template(INC_OPS_HEAD)
|
|
123
|
+
self.lite_ops_cc_template = template.Template(LITE_OPS_CC)
|
|
93
124
|
self.op_template = template.op_template
|
|
94
|
-
self.register_primitive_c_template = template.Template("
|
|
125
|
+
self.register_primitive_c_template = template.Template("${op_name}::${op_name}():BaseOperator(kName${op_name}) {}\n"
|
|
126
|
+
"REGISTER_PRIMITIVE_C(kName${op_name}, ${op_name});\n"
|
|
95
127
|
"MIND_API_OPERATOR_IMPL(${op_name}, BaseOperator);\n\n")
|
|
96
128
|
|
|
97
129
|
def generate(self, work_path, op_protos):
|
|
@@ -106,6 +138,7 @@ class LiteOpsCcGenerator(BaseGenerator):
|
|
|
106
138
|
None
|
|
107
139
|
"""
|
|
108
140
|
lite_ops_cc_gen_list = []
|
|
141
|
+
inc_ops_head_list = set()
|
|
109
142
|
for op_proto in op_protos:
|
|
110
143
|
arg_prim_init_str = ""
|
|
111
144
|
op_name = pyboost_utils.get_op_name(op_proto.op_name, op_proto.op_class.name)
|
|
@@ -120,9 +153,12 @@ class LiteOpsCcGenerator(BaseGenerator):
|
|
|
120
153
|
|
|
121
154
|
self.register_primitive_c_template.replace(op_name=op_name)
|
|
122
155
|
lite_ops_cc_gen_list.append(arg_prim_init_str + self.register_primitive_c_template.replace(op_name=op_name))
|
|
123
|
-
|
|
156
|
+
inc_ops_head_list.add(self.inc_ops_head_templat.replace(auto_gen_path=K.OP_DEF_AUTO_GENERATE_PATH,
|
|
157
|
+
ch=op_name[0].lower()))
|
|
158
|
+
sorted_inc_ops_head_str = sorted(inc_ops_head_list)
|
|
124
159
|
lite_ops_cc = self.lite_ops_cc_template.replace(auto_gen_path=K.OP_DEF_AUTO_GENERATE_PATH,
|
|
125
|
-
ops_namespace_body=lite_ops_cc_gen_list
|
|
160
|
+
ops_namespace_body=lite_ops_cc_gen_list,
|
|
161
|
+
inc_ops_head_str=sorted_inc_ops_head_str)
|
|
126
162
|
|
|
127
163
|
res_str = template.CC_LICENSE_STR + lite_ops_cc
|
|
128
164
|
save_path = os.path.join(work_path, K.MS_OP_DEF_AUTO_GENERATE_PATH)
|
|
@@ -19,14 +19,25 @@ Module for generating C++ operator definition files.
|
|
|
19
19
|
import os
|
|
20
20
|
import math
|
|
21
21
|
|
|
22
|
-
import gen_constants as K
|
|
23
|
-
import gen_utils
|
|
22
|
+
import common.gen_constants as K
|
|
23
|
+
import common.gen_utils as gen_utils
|
|
24
24
|
|
|
25
25
|
# refactored
|
|
26
|
-
from op_proto import OpProto
|
|
27
|
-
import template
|
|
26
|
+
from common.op_proto import OpProto
|
|
27
|
+
import common.template as template
|
|
28
28
|
|
|
29
|
-
from base_generator import BaseGenerator
|
|
29
|
+
from common.base_generator import BaseGenerator
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
CC_OPS_DEF = """
|
|
33
|
+
|
|
34
|
+
#include "$auto_generate_path/gen_ops_def.h"
|
|
35
|
+
#include "ir/signature.h"
|
|
36
|
+
$gen_include
|
|
37
|
+
|
|
38
|
+
namespace mindspore::ops {$gen_cc_code
|
|
39
|
+
} // namespace mindspore::ops
|
|
40
|
+
"""
|
|
30
41
|
|
|
31
42
|
|
|
32
43
|
class OpsDefCcGenerator(BaseGenerator):
|
|
@@ -43,7 +54,7 @@ class OpsDefCcGenerator(BaseGenerator):
|
|
|
43
54
|
self.empty_func_impl_declaration_template = template.Template("static OpFuncImpl g${class_name}FuncImpl;")
|
|
44
55
|
self.func_impl_define_template = template.Template("g${class_name}FuncImpl")
|
|
45
56
|
self.OP_PROTO_TEMPLATE = template.OP_PROTO_TEMPLATE
|
|
46
|
-
self.CC_OPS_DEF_TEMPLATE = template.Template(
|
|
57
|
+
self.CC_OPS_DEF_TEMPLATE = template.Template(CC_OPS_DEF)
|
|
47
58
|
|
|
48
59
|
def generate(self, work_path, op_protos):
|
|
49
60
|
"""
|
|
@@ -60,7 +71,7 @@ class OpsDefCcGenerator(BaseGenerator):
|
|
|
60
71
|
for op_proto in op_protos:
|
|
61
72
|
operator_name = op_proto.op_name
|
|
62
73
|
class_name = op_proto.op_class.name
|
|
63
|
-
if
|
|
74
|
+
if not op_proto.func_op:
|
|
64
75
|
gen_include_list.append(self.include_template.replace(path=K.MS_OPS_FUNC_IMPL_PATH,
|
|
65
76
|
operator_name=operator_name))
|
|
66
77
|
func_impl_declaration_str = self.func_impl_declaration_template.replace(class_name=class_name)
|
|
@@ -21,11 +21,11 @@ and any additional operators provided. This is useful for managing operator inte
|
|
|
21
21
|
|
|
22
22
|
import os
|
|
23
23
|
|
|
24
|
-
import template
|
|
25
|
-
from template import Template
|
|
26
|
-
from gen_utils import save_file
|
|
27
|
-
import gen_constants as K
|
|
28
|
-
from base_generator import BaseGenerator
|
|
24
|
+
import common.template as template
|
|
25
|
+
from common.template import Template
|
|
26
|
+
from common.gen_utils import save_file
|
|
27
|
+
import common.gen_constants as K
|
|
28
|
+
from common.base_generator import BaseGenerator
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
class OpsDefHGenerator(BaseGenerator):
|