mindspore 2.6.0__cp311-cp311-win_amd64.whl → 2.7.0rc1__cp311-cp311-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of mindspore might be problematic. Click here for more details.
- mindspore/.commit_id +1 -1
- mindspore/Microsoft.VisualStudio.Telemetry.dll +0 -0
- mindspore/Newtonsoft.Json.dll +0 -0
- mindspore/__init__.py +1 -1
- mindspore/_c_dataengine.cp311-win_amd64.pyd +0 -0
- mindspore/_c_expression.cp311-win_amd64.pyd +0 -0
- mindspore/_c_mindrecord.cp311-win_amd64.pyd +0 -0
- mindspore/_checkparam.py +40 -9
- mindspore/{_deprecated → _extends/optimize}/__init__.py +9 -3
- mindspore/_extends/optimize/cell_utils.py +96 -0
- mindspore/_extends/parse/__init__.py +2 -2
- mindspore/_extends/parse/compile_config.py +44 -22
- mindspore/_extends/parse/deprecated/deprecated_tensor_method.py +1 -1
- mindspore/_extends/parse/parser.py +36 -61
- mindspore/_extends/parse/resources.py +39 -0
- mindspore/_extends/parse/standard_method.py +32 -13
- mindspore/_extends/parse/trope.py +8 -1
- mindspore/_extends/pijit/__init__.py +1 -2
- mindspore/amp.py +4 -4
- mindspore/atlprov.dll +0 -0
- mindspore/avcodec-59.dll +0 -0
- mindspore/avdevice-59.dll +0 -0
- mindspore/avfilter-8.dll +0 -0
- mindspore/avformat-59.dll +0 -0
- mindspore/avutil-57.dll +0 -0
- mindspore/boost/adasum.py +1 -1
- mindspore/boost/boost_cell_wrapper.py +4 -4
- mindspore/c1.dll +0 -0
- mindspore/c1xx.dll +0 -0
- mindspore/c2.dll +0 -0
- mindspore/common/__init__.py +27 -2
- mindspore/common/_grad_function.py +2 -1
- mindspore/common/_pijit_context.py +28 -7
- mindspore/common/_stub_tensor.py +1 -209
- mindspore/common/_tensor_cpp_method.py +1 -1
- mindspore/common/_tensor_docs.py +76 -15
- mindspore/common/api.py +193 -112
- mindspore/common/dtype.py +21 -11
- mindspore/common/dump.py +10 -15
- mindspore/common/generator.py +2 -3
- mindspore/common/hook_handle.py +11 -2
- mindspore/common/jit_config.py +1 -1
- mindspore/common/jit_trace.py +84 -105
- mindspore/common/parameter.py +26 -12
- mindspore/common/recompute.py +3 -3
- mindspore/common/sparse_tensor.py +0 -3
- mindspore/common/symbol.py +0 -1
- mindspore/common/tensor.py +48 -83
- mindspore/communication/_comm_helper.py +46 -4
- mindspore/communication/management.py +79 -7
- mindspore/context.py +38 -23
- mindspore/dataset/core/config.py +3 -3
- mindspore/dataset/engine/datasets.py +20 -7
- mindspore/dataset/engine/datasets_user_defined.py +32 -2
- mindspore/dataset/engine/iterators.py +2 -2
- mindspore/dataset/engine/obs/config_loader.py +2 -2
- mindspore/dataset/engine/obs/obs_mindrecord_dataset.py +8 -0
- mindspore/dataset/transforms/py_transforms.py +7 -3
- mindspore/dataset/transforms/transforms.py +7 -3
- mindspore/dataset/vision/validators.py +1 -0
- mindspore/device_context/ascend/device.py +1 -1
- mindspore/device_context/gpu/__init__.py +2 -2
- mindspore/device_context/gpu/device.py +1 -1
- mindspore/device_context/gpu/op_precision.py +4 -2
- mindspore/device_context/gpu/op_tuning.py +6 -3
- mindspore/device_manager.py +16 -9
- mindspore/dnnl.dll +0 -0
- mindspore/dpcmi.dll +0 -0
- mindspore/experimental/llm_boost/ascend_native/llama_boost_ascend_native.py +3 -5
- mindspore/experimental/llm_boost/atb/boost_base.py +2 -3
- mindspore/experimental/optim/adadelta.py +13 -20
- mindspore/experimental/optim/adagrad.py +15 -22
- mindspore/experimental/optim/adam.py +17 -24
- mindspore/experimental/optim/adamax.py +14 -22
- mindspore/experimental/optim/adamw.py +28 -34
- mindspore/experimental/optim/asgd.py +15 -25
- mindspore/experimental/optim/lr_scheduler.py +27 -45
- mindspore/experimental/optim/nadam.py +14 -24
- mindspore/experimental/optim/optimizer.py +13 -23
- mindspore/experimental/optim/radam.py +18 -24
- mindspore/experimental/optim/rmsprop.py +14 -25
- mindspore/experimental/optim/rprop.py +15 -26
- mindspore/experimental/optim/sgd.py +9 -19
- mindspore/hal/__init__.py +4 -4
- mindspore/hal/contiguous_tensors_handle.py +2 -2
- mindspore/hal/memory.py +1 -0
- mindspore/include/api/cell.h +37 -1
- mindspore/include/api/delegate.h +10 -0
- mindspore/include/api/model.h +3 -0
- mindspore/include/api/types.h +2 -2
- mindspore/include/c_api/model_c.h +0 -58
- mindspore/include/c_api/tensor_c.h +0 -26
- mindspore/include/dataset/vision_ascend.h +1 -1
- mindspore/jpeg62.dll +0 -0
- mindspore/mindrecord/tools/cifar10.py +60 -11
- mindspore/mindrecord/tools/cifar10_to_mr.py +5 -0
- mindspore/mindspore_backend_common.dll +0 -0
- mindspore/mindspore_backend_manager.dll +0 -0
- mindspore/mindspore_common.dll +0 -0
- mindspore/mindspore_core.dll +0 -0
- mindspore/mindspore_cpu_res_manager.dll +0 -0
- mindspore/mindspore_dump.dll +0 -0
- mindspore/mindspore_frontend.dll +0 -0
- mindspore/mindspore_glog.dll +0 -0
- mindspore/mindspore_memory_pool.dll +0 -0
- mindspore/mindspore_ms_backend.dll +0 -0
- mindspore/mindspore_ops.dll +0 -0
- mindspore/mindspore_ops_host.dll +0 -0
- mindspore/mindspore_ops_kernel_common.dll +0 -0
- mindspore/mindspore_profiler.dll +0 -0
- mindspore/mindspore_pyboost.dll +0 -0
- mindspore/mindspore_pynative.dll +0 -0
- mindspore/mindspore_res_manager.dll +0 -0
- mindspore/mindspore_runtime_pipeline.dll +0 -0
- mindspore/mint/__init__.py +4 -44
- mindspore/mint/distributed/__init__.py +1 -0
- mindspore/mint/distributed/distributed.py +208 -5
- mindspore/mint/nn/__init__.py +1 -1
- mindspore/mint/nn/functional.py +53 -6
- mindspore/mint/nn/layer/_functions.py +164 -294
- mindspore/mint/nn/layer/activation.py +8 -6
- mindspore/mint/nn/layer/conv.py +122 -98
- mindspore/mint/nn/layer/normalization.py +8 -22
- mindspore/mint/optim/adam.py +19 -18
- mindspore/mint/optim/adamw.py +14 -8
- mindspore/mint/optim/sgd.py +5 -5
- mindspore/msobj140.dll +0 -0
- mindspore/mspdb140.dll +0 -0
- mindspore/mspdbcore.dll +0 -0
- mindspore/mspdbst.dll +0 -0
- mindspore/mspft140.dll +0 -0
- mindspore/msvcdis140.dll +0 -0
- mindspore/msvcp140_1.dll +0 -0
- mindspore/msvcp140_2.dll +0 -0
- mindspore/msvcp140_atomic_wait.dll +0 -0
- mindspore/msvcp140_codecvt_ids.dll +0 -0
- mindspore/nn/cell.py +325 -499
- mindspore/nn/grad/cell_grad.py +11 -12
- mindspore/nn/layer/activation.py +32 -34
- mindspore/nn/layer/basic.py +67 -64
- mindspore/nn/layer/channel_shuffle.py +4 -4
- mindspore/nn/layer/combined.py +4 -2
- mindspore/nn/layer/conv.py +86 -85
- mindspore/nn/layer/dense.py +9 -7
- mindspore/nn/layer/embedding.py +50 -52
- mindspore/nn/layer/image.py +37 -39
- mindspore/nn/layer/math.py +111 -112
- mindspore/nn/layer/normalization.py +56 -44
- mindspore/nn/layer/pooling.py +58 -63
- mindspore/nn/layer/rnn_cells.py +33 -33
- mindspore/nn/layer/rnns.py +56 -56
- mindspore/nn/layer/thor_layer.py +74 -73
- mindspore/nn/layer/transformer.py +11 -1
- mindspore/nn/learning_rate_schedule.py +20 -20
- mindspore/nn/loss/loss.py +79 -81
- mindspore/nn/optim/adam.py +1 -1
- mindspore/nn/optim/adasum.py +2 -2
- mindspore/nn/optim/optimizer.py +1 -1
- mindspore/nn/optim/thor.py +2 -2
- mindspore/nn/probability/distribution/exponential.py +2 -1
- mindspore/nn/probability/distribution/poisson.py +2 -1
- mindspore/nn/sparse/sparse.py +3 -3
- mindspore/nn/wrap/cell_wrapper.py +34 -37
- mindspore/nn/wrap/grad_reducer.py +37 -37
- mindspore/nn/wrap/loss_scale.py +72 -74
- mindspore/numpy/array_creations.py +5 -5
- mindspore/numpy/fft.py +1 -1
- mindspore/numpy/math_ops.py +1 -1
- mindspore/opencv_core452.dll +0 -0
- mindspore/opencv_imgcodecs452.dll +0 -0
- mindspore/opencv_imgproc452.dll +0 -0
- mindspore/ops/_grad_experimental/grad_comm_ops.py +51 -13
- mindspore/ops/_grad_experimental/grad_debug_ops.py +14 -0
- mindspore/ops/_vmap/vmap_array_ops.py +6 -13
- mindspore/ops/_vmap/vmap_nn_ops.py +8 -16
- mindspore/ops/auto_generate/cpp_create_prim_instance_helper.py +17 -8
- mindspore/ops/auto_generate/gen_extend_func.py +1 -51
- mindspore/ops/auto_generate/gen_ops_def.py +463 -257
- mindspore/ops/auto_generate/gen_ops_prim.py +1127 -885
- mindspore/ops/auto_generate/pyboost_inner_prim.py +31 -1
- mindspore/ops/composite/__init__.py +10 -0
- mindspore/ops/composite/base.py +8 -4
- mindspore/ops/composite/multitype_ops/__init__.py +12 -1
- mindspore/ops/composite/multitype_ops/_compile_utils.py +132 -108
- mindspore/ops/composite/multitype_ops/add_impl.py +70 -2
- mindspore/ops/composite/multitype_ops/div_impl.py +49 -0
- mindspore/ops/composite/multitype_ops/floordiv_impl.py +29 -0
- mindspore/ops/composite/multitype_ops/getitem_impl.py +11 -0
- mindspore/ops/composite/multitype_ops/mod_impl.py +5 -3
- mindspore/ops/composite/multitype_ops/mul_impl.py +49 -0
- mindspore/ops/composite/multitype_ops/setitem_impl.py +57 -0
- mindspore/ops/composite/multitype_ops/sub_impl.py +34 -0
- mindspore/ops/composite/multitype_ops/zeros_like_impl.py +14 -0
- mindspore/ops/function/__init__.py +3 -1
- mindspore/ops/function/_add_attr_func.py +11 -6
- mindspore/ops/function/array_func.py +7 -94
- mindspore/ops/function/debug_func.py +4 -3
- mindspore/ops/function/grad/grad_func.py +1 -1
- mindspore/ops/function/math_func.py +21 -367
- mindspore/ops/function/nn_func.py +26 -41
- mindspore/ops/function/other_func.py +4 -1
- mindspore/ops/function/random_func.py +31 -4
- mindspore/ops/functional.py +0 -2
- mindspore/ops/functional_overload.py +463 -6
- mindspore/ops/op_info_register.py +21 -0
- mindspore/ops/operations/__init__.py +5 -2
- mindspore/ops/operations/_custom_ops_utils.py +675 -8
- mindspore/ops/operations/_inner_ops.py +3 -6
- mindspore/ops/operations/_sequence_ops.py +1 -1
- mindspore/ops/operations/comm_ops.py +185 -26
- mindspore/ops/operations/custom_ops.py +235 -172
- mindspore/ops/operations/debug_ops.py +55 -4
- mindspore/ops/operations/image_ops.py +13 -13
- mindspore/ops/operations/manually_defined/ops_def.py +15 -16
- mindspore/ops/operations/math_ops.py +3 -4
- mindspore/ops/operations/nn_ops.py +5 -6
- mindspore/ops/primitive.py +6 -10
- mindspore/ops/tensor_method.py +36 -4
- mindspore/ops_generate/api/cpp_create_prim_instance_helper_generator.py +1 -1
- mindspore/ops_generate/api/functional_map_cpp_generator.py +10 -9
- mindspore/ops_generate/api/functions_cc_generator.py +58 -10
- mindspore/ops_generate/api/tensor_func_reg_cpp_generator.py +1 -1
- mindspore/ops_generate/common/base_generator.py +14 -0
- mindspore/ops_generate/common/gen_constants.py +7 -2
- mindspore/ops_generate/common/gen_utils.py +0 -19
- mindspore/ops_generate/common/op_proto.py +11 -4
- mindspore/ops_generate/common/template.py +88 -11
- mindspore/ops_generate/gen_ops.py +1 -1
- mindspore/ops_generate/op_def/lite_ops_cpp_generator.py +4 -4
- mindspore/ops_generate/op_def/ops_name_h_generator.py +0 -3
- mindspore/ops_generate/op_def/ops_primitive_h_generator.py +0 -4
- mindspore/ops_generate/op_def_py/op_prim_py_generator.py +5 -2
- mindspore/ops_generate/pyboost/auto_grad_impl_cc_generator.py +49 -8
- mindspore/ops_generate/pyboost/auto_grad_reg_cc_generator.py +2 -2
- mindspore/ops_generate/pyboost/gen_pyboost_func.py +31 -0
- mindspore/ops_generate/pyboost/op_template_parser.py +98 -72
- mindspore/ops_generate/pyboost/pyboost_functions_cpp_generator.py +70 -273
- mindspore/ops_generate/pyboost/pyboost_functions_h_generator.py +14 -6
- mindspore/ops_generate/pyboost/pyboost_functions_impl_cpp_generator.py +316 -0
- mindspore/ops_generate/pyboost/pyboost_functions_py_generator.py +1 -1
- mindspore/ops_generate/pyboost/pyboost_grad_function_cpp_generator.py +5 -3
- mindspore/ops_generate/pyboost/pyboost_inner_prim_generator.py +1 -1
- mindspore/ops_generate/pyboost/pyboost_internal_functions_cpp_generator.py +76 -0
- mindspore/ops_generate/pyboost/pyboost_internal_functions_h_generator.py +76 -0
- mindspore/ops_generate/pyboost/pyboost_internal_kernel_info_adapter_generator.py +125 -0
- mindspore/ops_generate/pyboost/pyboost_native_grad_functions_generator.py +4 -3
- mindspore/ops_generate/pyboost/pyboost_op_cpp_code_generator.py +348 -61
- mindspore/ops_generate/pyboost/pyboost_overload_functions_cpp_generator.py +1 -1
- mindspore/ops_generate/pyboost/pyboost_utils.py +118 -9
- mindspore/ops_generate/tensor_py_cc_generator.py +1 -24
- mindspore/parallel/_auto_parallel_context.py +4 -2
- mindspore/parallel/_cell_wrapper.py +106 -40
- mindspore/parallel/_parallel_serialization.py +1 -1
- mindspore/parallel/_ps_context.py +4 -6
- mindspore/parallel/_tensor.py +167 -12
- mindspore/parallel/_transformer/moe.py +1 -1
- mindspore/parallel/_transformer/transformer.py +13 -8
- mindspore/parallel/auto_parallel.py +12 -5
- mindspore/parallel/checkpoint_convert.py +3 -3
- mindspore/parallel/checkpoint_transform.py +3 -1
- mindspore/parallel/cluster/process_entity/_api.py +84 -48
- mindspore/parallel/cluster/process_entity/_utils.py +95 -7
- mindspore/parallel/cluster/run.py +43 -4
- mindspore/parallel/function/__init__.py +8 -1
- mindspore/parallel/function/reshard_func.py +1 -1
- mindspore/parallel/nn/__init__.py +15 -2
- mindspore/parallel/nn/parallel_cell_wrapper.py +9 -10
- mindspore/parallel/nn/parallel_grad_reducer.py +7 -6
- mindspore/parallel/shard.py +2 -2
- mindspore/parallel/transform_safetensors.py +462 -174
- mindspore/pgodb140.dll +0 -0
- mindspore/pgort140.dll +0 -0
- mindspore/profiler/__init__.py +2 -1
- mindspore/profiler/analysis/parser/timeline_assembly_factory/ascend_timeline_assembler.py +7 -7
- mindspore/profiler/analysis/parser/timeline_assembly_factory/base_timeline_assembler.py +3 -0
- mindspore/profiler/analysis/parser/timeline_assembly_factory/trace_view_container.py +3 -0
- mindspore/profiler/analysis/parser/timeline_creator/cpu_op_timeline_creator.py +3 -3
- mindspore/profiler/analysis/parser/timeline_creator/fwk_timeline_creator.py +3 -3
- mindspore/profiler/analysis/parser/timeline_creator/msprof_timeline_creator.py +4 -4
- mindspore/profiler/analysis/parser/timeline_creator/scope_layer_timeline_creator.py +3 -3
- mindspore/profiler/analysis/parser/timeline_event/fwk_event.py +4 -1
- mindspore/profiler/analysis/parser/timeline_event/timeline_event_pool.py +2 -1
- mindspore/profiler/analysis/task_manager.py +1 -1
- mindspore/profiler/analysis/viewer/ascend_communication_viewer.py +5 -1
- mindspore/profiler/analysis/viewer/ascend_integrate_viewer.py +2 -1
- mindspore/profiler/analysis/viewer/ascend_op_memory_viewer.py +42 -22
- mindspore/profiler/analysis/viewer/ascend_step_trace_time_viewer.py +3 -2
- mindspore/profiler/analysis/viewer/ms_minddata_viewer.py +9 -5
- mindspore/profiler/analysis/viewer/ms_operator_details_viewer.py +132 -0
- mindspore/profiler/common/constant.py +16 -0
- mindspore/profiler/common/profiler_context.py +25 -27
- mindspore/profiler/common/profiler_info.py +0 -16
- mindspore/profiler/common/profiler_op_analyse.py +235 -0
- mindspore/profiler/common/profiler_output_path.py +23 -8
- mindspore/profiler/common/profiler_parameters.py +128 -35
- mindspore/profiler/dynamic_profile/__init__.py +0 -0
- mindspore/profiler/dynamic_profile/dynamic_monitor_proxy.py +39 -0
- mindspore/profiler/dynamic_profile/dynamic_profiler_config_context.py +666 -0
- mindspore/profiler/dynamic_profile/dynamic_profiler_utils.py +62 -0
- mindspore/profiler/dynamic_profiler.py +305 -314
- mindspore/profiler/envprofiler.py +12 -7
- mindspore/profiler/experimental_config.py +96 -6
- mindspore/profiler/mstx.py +33 -12
- mindspore/profiler/platform/__init__.py +2 -3
- mindspore/profiler/platform/npu_profiler.py +29 -19
- mindspore/profiler/profiler.py +35 -19
- mindspore/profiler/profiler_action_controller.py +64 -76
- mindspore/profiler/schedule.py +10 -4
- mindspore/rewrite/common/config.py +1 -0
- mindspore/rewrite/common/namer.py +1 -0
- mindspore/rewrite/common/namespace.py +1 -0
- mindspore/rewrite/node/node.py +31 -11
- mindspore/rewrite/parsers/assign_parser.py +1 -1
- mindspore/rewrite/symbol_tree/symbol_tree.py +1 -1
- mindspore/run_check/_check_version.py +7 -10
- mindspore/runtime/__init__.py +5 -5
- mindspore/runtime/event.py +10 -4
- mindspore/runtime/executor.py +60 -45
- mindspore/runtime/memory.py +21 -30
- mindspore/runtime/thread_bind_core.py +298 -164
- mindspore/safeguard/rewrite_obfuscation.py +12 -13
- mindspore/swresample-4.dll +0 -0
- mindspore/swscale-6.dll +0 -0
- mindspore/tbbmalloc.dll +0 -0
- mindspore/tinyxml2.dll +0 -0
- mindspore/train/_utils.py +6 -2
- mindspore/train/amp.py +43 -20
- mindspore/train/callback/__init__.py +5 -5
- mindspore/train/callback/_checkpoint.py +3 -6
- mindspore/train/callback/_flops_collector.py +1 -1
- mindspore/train/callback/_landscape.py +0 -1
- mindspore/train/callback/_train_fault_tolerance.py +71 -13
- mindspore/train/data_sink.py +11 -2
- mindspore/train/dataset_helper.py +9 -0
- mindspore/train/model.py +51 -33
- mindspore/train/serialization.py +133 -111
- mindspore/train/summary/summary_record.py +13 -2
- mindspore/turbojpeg.dll +0 -0
- mindspore/utils/__init__.py +3 -2
- mindspore/utils/dryrun.py +0 -6
- mindspore/utils/runtime_execution_order_check.py +162 -78
- mindspore/utils/sdc_detect.py +68 -0
- mindspore/utils/utils.py +6 -9
- mindspore/vcmeta.dll +0 -0
- mindspore/vcruntime140.dll +0 -0
- mindspore/vcruntime140_1.dll +0 -0
- mindspore/version.py +1 -1
- {mindspore-2.6.0.dist-info → mindspore-2.7.0rc1.dist-info}/METADATA +5 -4
- {mindspore-2.6.0.dist-info → mindspore-2.7.0rc1.dist-info}/RECORD +352 -390
- mindspore/_deprecated/jit.py +0 -198
- mindspore/experimental/es/__init__.py +0 -22
- mindspore/experimental/es/embedding_service.py +0 -891
- mindspore/experimental/es/embedding_service_layer.py +0 -581
- mindspore/profiler/parser/__init__.py +0 -14
- mindspore/profiler/parser/aicpu_data_parser.py +0 -272
- mindspore/profiler/parser/ascend_analysis/__init__.py +0 -14
- mindspore/profiler/parser/ascend_analysis/constant.py +0 -71
- mindspore/profiler/parser/ascend_analysis/file_manager.py +0 -180
- mindspore/profiler/parser/ascend_analysis/function_event.py +0 -185
- mindspore/profiler/parser/ascend_analysis/fwk_cann_parser.py +0 -136
- mindspore/profiler/parser/ascend_analysis/fwk_file_parser.py +0 -131
- mindspore/profiler/parser/ascend_analysis/msprof_timeline_parser.py +0 -104
- mindspore/profiler/parser/ascend_analysis/path_manager.py +0 -313
- mindspore/profiler/parser/ascend_analysis/profiler_info_parser.py +0 -123
- mindspore/profiler/parser/ascend_analysis/tlv_decoder.py +0 -86
- mindspore/profiler/parser/ascend_analysis/trace_event_manager.py +0 -75
- mindspore/profiler/parser/ascend_cluster_generator.py +0 -116
- mindspore/profiler/parser/ascend_communicate_generator.py +0 -314
- mindspore/profiler/parser/ascend_flops_generator.py +0 -116
- mindspore/profiler/parser/ascend_fpbp_generator.py +0 -82
- mindspore/profiler/parser/ascend_hccl_generator.py +0 -271
- mindspore/profiler/parser/ascend_integrate_generator.py +0 -42
- mindspore/profiler/parser/ascend_memory_generator.py +0 -185
- mindspore/profiler/parser/ascend_msprof_exporter.py +0 -282
- mindspore/profiler/parser/ascend_msprof_generator.py +0 -187
- mindspore/profiler/parser/ascend_op_generator.py +0 -334
- mindspore/profiler/parser/ascend_steptrace_generator.py +0 -94
- mindspore/profiler/parser/ascend_timeline_generator.py +0 -545
- mindspore/profiler/parser/base_timeline_generator.py +0 -483
- mindspore/profiler/parser/container.py +0 -229
- mindspore/profiler/parser/cpu_gpu_timeline_generator.py +0 -697
- mindspore/profiler/parser/flops_parser.py +0 -531
- mindspore/profiler/parser/framework_enum.py +0 -111
- mindspore/profiler/parser/framework_parser.py +0 -464
- mindspore/profiler/parser/framework_struct.py +0 -61
- mindspore/profiler/parser/gpu_analysis/__init__.py +0 -14
- mindspore/profiler/parser/gpu_analysis/function_event.py +0 -44
- mindspore/profiler/parser/gpu_analysis/fwk_file_parser.py +0 -89
- mindspore/profiler/parser/gpu_analysis/profiler_info_parser.py +0 -72
- mindspore/profiler/parser/hccl_parser.py +0 -573
- mindspore/profiler/parser/hwts_log_parser.py +0 -122
- mindspore/profiler/parser/integrator.py +0 -526
- mindspore/profiler/parser/memory_usage_parser.py +0 -277
- mindspore/profiler/parser/minddata_analyzer.py +0 -800
- mindspore/profiler/parser/minddata_parser.py +0 -186
- mindspore/profiler/parser/minddata_pipeline_parser.py +0 -299
- mindspore/profiler/parser/op_intermediate_parser.py +0 -149
- mindspore/profiler/parser/optime_parser.py +0 -250
- mindspore/profiler/parser/profiler_info.py +0 -213
- mindspore/profiler/parser/step_trace_parser.py +0 -666
- {mindspore-2.6.0.dist-info → mindspore-2.7.0rc1.dist-info}/WHEEL +0 -0
- {mindspore-2.6.0.dist-info → mindspore-2.7.0rc1.dist-info}/entry_points.txt +0 -0
- {mindspore-2.6.0.dist-info → mindspore-2.7.0rc1.dist-info}/top_level.txt +0 -0
|
@@ -64,7 +64,6 @@ class ProfilerInfo:
|
|
|
64
64
|
ANALYSIS_COST_TIME = "analysis_cost_time"
|
|
65
65
|
MS_VERSION = "ms_version"
|
|
66
66
|
CANN_VERSION = "cann_version"
|
|
67
|
-
CONTEXT_MODE = "context_mode"
|
|
68
67
|
JIT_LEVEL = "jit_level"
|
|
69
68
|
|
|
70
69
|
US_TO_NS = 1000
|
|
@@ -77,7 +76,6 @@ class ProfilerInfo:
|
|
|
77
76
|
self.ANALYSIS_COST_TIME: {},
|
|
78
77
|
self.MS_VERSION: ms_version,
|
|
79
78
|
self.CANN_VERSION: get_cann_version(),
|
|
80
|
-
self.CONTEXT_MODE: -1,
|
|
81
79
|
self.JIT_LEVEL: "",
|
|
82
80
|
}
|
|
83
81
|
# time params
|
|
@@ -132,20 +130,6 @@ class ProfilerInfo:
|
|
|
132
130
|
"""
|
|
133
131
|
self._profiler_info[self.PROFILER_PARAMETERS] = value
|
|
134
132
|
|
|
135
|
-
@property
|
|
136
|
-
def context_mode(self) -> int:
|
|
137
|
-
"""
|
|
138
|
-
Get context mode.
|
|
139
|
-
"""
|
|
140
|
-
return self._profiler_info[self.CONTEXT_MODE]
|
|
141
|
-
|
|
142
|
-
@context_mode.setter
|
|
143
|
-
def context_mode(self, value: int):
|
|
144
|
-
"""
|
|
145
|
-
Set context mode.
|
|
146
|
-
"""
|
|
147
|
-
self._profiler_info[self.CONTEXT_MODE] = value
|
|
148
|
-
|
|
149
133
|
@property
|
|
150
134
|
def jit_level(self) -> str:
|
|
151
135
|
"""
|
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
# Copyright 2024-2025 Huawei Technologies Co., Ltd
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
# ============================================================================
|
|
15
|
+
"""The OpAnalyser."""
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
from collections import defaultdict
|
|
19
|
+
from mindspore.profiler.common.validator.validate_path import validate_and_normalize_path
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class OpAnalyser:
|
|
23
|
+
"""
|
|
24
|
+
The parser for parsing framework files.
|
|
25
|
+
|
|
26
|
+
Note:
|
|
27
|
+
This parser only supports CPU and GPU devices.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
output_path (str): The profiling path which should contain GPU profiling data.
|
|
31
|
+
dev_id (str): The device ID.
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
def __init__(self, output_path, dev_id, op_names=None):
|
|
35
|
+
"""The parser for parsing framework files."""
|
|
36
|
+
self._dev_id = dev_id
|
|
37
|
+
self._output_path = output_path
|
|
38
|
+
self.op_names = op_names
|
|
39
|
+
self.op_name = ''
|
|
40
|
+
self.framework_list = []
|
|
41
|
+
self.op_detail = {}
|
|
42
|
+
self.operation_info = {}
|
|
43
|
+
self.activity_info_dir = []
|
|
44
|
+
self.framework_info_dir = []
|
|
45
|
+
self.cpu_detail_info_dir = []
|
|
46
|
+
self.gpu_op_type_info_dir = []
|
|
47
|
+
self.op_execute_times = {}
|
|
48
|
+
self.op_step_shape_info = defaultdict(list)
|
|
49
|
+
|
|
50
|
+
def parse(self):
|
|
51
|
+
"""Parse op performance data."""
|
|
52
|
+
self.get_device_target_filename()
|
|
53
|
+
self.get_framework_summary()
|
|
54
|
+
self.get_cpu_op_detail_info()
|
|
55
|
+
self.get_activity_op_info()
|
|
56
|
+
if isinstance(self.op_names, str):
|
|
57
|
+
self.combine_performance_data(self.op_names)
|
|
58
|
+
elif isinstance(self.op_names, list):
|
|
59
|
+
for op_name in self.op_names:
|
|
60
|
+
self.combine_performance_data(op_name)
|
|
61
|
+
self.operation_info["device_id"] = self._dev_id
|
|
62
|
+
return json.dumps(self.operation_info)
|
|
63
|
+
|
|
64
|
+
def get_framework_summary(self):
|
|
65
|
+
"""Get framework data."""
|
|
66
|
+
for filename in self.framework_info_dir:
|
|
67
|
+
op_side = filename.split('_')[0]
|
|
68
|
+
framework_file_path = os.path.join(self._output_path, filename)
|
|
69
|
+
framework_file_path = validate_and_normalize_path(framework_file_path)
|
|
70
|
+
with open(framework_file_path, 'r') as f_obj:
|
|
71
|
+
framework_info = f_obj.readlines()
|
|
72
|
+
for line_info in framework_info:
|
|
73
|
+
line_info = line_info.strip(' ').strip('\n').split(';')
|
|
74
|
+
# line_info[0]: op_type, line_info[1]: op_name, line_info[2]: graph_id, line_info[3]: input_shape;
|
|
75
|
+
input_shape = line_info[3:]
|
|
76
|
+
item = [line_info[0], line_info[1], input_shape, op_side]
|
|
77
|
+
if not self.op_step_shape_info.get(line_info[1]):
|
|
78
|
+
self.op_step_shape_info[line_info[1]].append(op_side)
|
|
79
|
+
self.op_step_shape_info[line_info[1]].append(input_shape)
|
|
80
|
+
if item not in self.framework_list:
|
|
81
|
+
self.framework_list.append(item)
|
|
82
|
+
|
|
83
|
+
def get_cpu_op_detail_info(self):
|
|
84
|
+
"""Get cpu operators detail data."""
|
|
85
|
+
for filename in self.cpu_detail_info_dir:
|
|
86
|
+
op_side = filename.split('_')[0]
|
|
87
|
+
op_detail_file_path = os.path.join(self._output_path, filename)
|
|
88
|
+
op_detail_file_path = validate_and_normalize_path(op_detail_file_path)
|
|
89
|
+
with open(op_detail_file_path, 'r') as f_obj:
|
|
90
|
+
op_detail_info = f_obj.readlines()
|
|
91
|
+
for line_info in op_detail_info[1:]:
|
|
92
|
+
line_info = line_info.strip(' ').strip('\n').split(',')
|
|
93
|
+
if not self.op_detail.get(line_info[2]):
|
|
94
|
+
# line_info[4]: op_occurrences, line_info[5]: op_detail_time(us), line_info[6]: op_avg_time(us);
|
|
95
|
+
self.op_detail[line_info[2]] = [float(line_info[4]), float(line_info[5]),
|
|
96
|
+
float(line_info[6]), op_side]
|
|
97
|
+
|
|
98
|
+
def get_execute_times(self):
|
|
99
|
+
"""Get gpu operators execute times."""
|
|
100
|
+
if self.gpu_op_type_info_dir:
|
|
101
|
+
gpu_op_type_file_path = os.path.join(self._output_path, self.gpu_op_type_info_dir[0])
|
|
102
|
+
gpu_op_type_file_path = validate_and_normalize_path(gpu_op_type_file_path)
|
|
103
|
+
with open(gpu_op_type_file_path, 'r') as fp:
|
|
104
|
+
op_type_info = fp.readlines()
|
|
105
|
+
for line_info in op_type_info[1:]:
|
|
106
|
+
line_info = line_info.strip(' ').strip('\n').split(',')
|
|
107
|
+
self.op_execute_times[line_info[0]] = line_info[1]
|
|
108
|
+
|
|
109
|
+
def get_activity_op_info(self):
|
|
110
|
+
"""Get op detail data."""
|
|
111
|
+
all_file = os.listdir(self._output_path)
|
|
112
|
+
for file_name in all_file:
|
|
113
|
+
if file_name.startswith('gpu_op_type') and file_name.endswith(f'{self._dev_id}.csv'):
|
|
114
|
+
self.gpu_op_type_info_dir.append(file_name)
|
|
115
|
+
if not self.gpu_op_type_info_dir and self.activity_info_dir:
|
|
116
|
+
raise RuntimeError(f'The output file <%s> is not found.' % self.gpu_op_type_info_dir)
|
|
117
|
+
self.get_execute_times()
|
|
118
|
+
for filename in self.activity_info_dir:
|
|
119
|
+
op_side = filename.split('_')[0]
|
|
120
|
+
activity_file_path = os.path.join(self._output_path, filename)
|
|
121
|
+
activity_file_path = validate_and_normalize_path(activity_file_path)
|
|
122
|
+
with open(activity_file_path, 'r') as file:
|
|
123
|
+
activity_info = file.readlines()
|
|
124
|
+
for line_info in activity_info[1:]:
|
|
125
|
+
line_info = line_info.strip(' ').strip('\n').replace(', ', ';').split(',')
|
|
126
|
+
op_name = line_info[2].split('/')[-1]
|
|
127
|
+
# op_name: xxx-opx
|
|
128
|
+
op_type = op_name.split('-')[0]
|
|
129
|
+
op_value = self.op_execute_times.get(op_type)
|
|
130
|
+
if op_value is not None and op_value != '':
|
|
131
|
+
try:
|
|
132
|
+
op_occurrences = int(op_value)
|
|
133
|
+
except (ValueError, TypeError):
|
|
134
|
+
op_occurrences = 1
|
|
135
|
+
else:
|
|
136
|
+
op_occurrences = 1
|
|
137
|
+
|
|
138
|
+
op_total_time = float(line_info[-4])
|
|
139
|
+
if not self.op_detail.get(op_name):
|
|
140
|
+
# line_info[4]: op_occurrences, line_info[5]: op_detail_time(us), line_info[6]: op_avg_time(us);
|
|
141
|
+
if op_occurrences > 0:
|
|
142
|
+
avg_time = round(op_total_time / op_occurrences, 4)
|
|
143
|
+
else:
|
|
144
|
+
avg_time = 0
|
|
145
|
+
self.op_detail[op_name] = [
|
|
146
|
+
op_occurrences, op_total_time, avg_time, op_side
|
|
147
|
+
]
|
|
148
|
+
else:
|
|
149
|
+
self.op_detail.get(op_name)[1] += op_total_time
|
|
150
|
+
self.op_detail.get(op_name)[2] = self.op_detail.get(op_name)[1] / self.op_detail.get(op_name)[0]
|
|
151
|
+
self.op_detail[op_name] = [
|
|
152
|
+
self.op_detail.get(op_name)[0],
|
|
153
|
+
round(self.op_detail.get(op_name)[1], 4),
|
|
154
|
+
round(self.op_detail.get(op_name)[2], 4), op_side
|
|
155
|
+
]
|
|
156
|
+
|
|
157
|
+
def combine_performance_data(self, op_name):
|
|
158
|
+
"""Combine operator detail info with framework info."""
|
|
159
|
+
unique_op_info = []
|
|
160
|
+
op_shape_dict = {}
|
|
161
|
+
operation_info = {}
|
|
162
|
+
factor = 1000 # convert time unit from ms to us.
|
|
163
|
+
for line_info in self.framework_list:
|
|
164
|
+
op_detail = self.op_detail.get(line_info[1])
|
|
165
|
+
if not op_detail:
|
|
166
|
+
continue
|
|
167
|
+
if op_name in line_info and line_info[3] == op_detail[3]:
|
|
168
|
+
op_side = line_info[3]
|
|
169
|
+
op_shape = '[{}]{}'.format(op_side, ','.join(line_info[2]))
|
|
170
|
+
op_occurrences = int(op_detail[0])
|
|
171
|
+
op_total_time = float(op_detail[1])
|
|
172
|
+
op_avg_time = float(op_detail[2])
|
|
173
|
+
if op_shape in op_shape_dict:
|
|
174
|
+
# Classify according to the operator information of the same shape.
|
|
175
|
+
op_shape_dict.get(op_shape)[0] += op_occurrences
|
|
176
|
+
op_shape_dict.get(op_shape)[1] += op_total_time
|
|
177
|
+
if op_shape_dict.get(op_shape)[0] > 0:
|
|
178
|
+
op_shape_dict.get(op_shape)[2] = op_shape_dict.get(op_shape)[1] / op_shape_dict.get(op_shape)[0]
|
|
179
|
+
else:
|
|
180
|
+
op_shape_dict.get(op_shape)[2] = 0
|
|
181
|
+
op_shape_dict[op_shape] = [
|
|
182
|
+
op_shape_dict.get(op_shape)[0], round(op_shape_dict.get(op_shape)[1], 4),
|
|
183
|
+
round(op_shape_dict.get(op_shape)[2], 4), op_side
|
|
184
|
+
]
|
|
185
|
+
else:
|
|
186
|
+
op_shape_dict[op_shape] = [op_occurrences, op_total_time, op_avg_time, op_side]
|
|
187
|
+
|
|
188
|
+
for input_shape in op_shape_dict:
|
|
189
|
+
# 0: op_occurrences, 1: op_total_time, 2: op_avg_time, 3: op_side
|
|
190
|
+
operation_info['op_side'] = op_shape_dict.get(input_shape)[3]
|
|
191
|
+
operation_info['input_shape'] = input_shape.strip('[').split(']')[-1]
|
|
192
|
+
operation_info['op_occurrences'] = op_shape_dict.get(input_shape)[0]
|
|
193
|
+
if operation_info.get('op_side') == 'cpu':
|
|
194
|
+
operation_info['op_total_time(us)'] = round(op_shape_dict.get(input_shape)[1] * factor, 4)
|
|
195
|
+
operation_info['op_avg_time(us)'] = round(op_shape_dict.get(input_shape)[2] * factor, 4)
|
|
196
|
+
else:
|
|
197
|
+
operation_info['op_total_time(us)'] = op_shape_dict.get(input_shape)[1]
|
|
198
|
+
operation_info['op_avg_time(us)'] = op_shape_dict.get(input_shape)[2]
|
|
199
|
+
unique_op_info.append(operation_info)
|
|
200
|
+
operation_info = dict()
|
|
201
|
+
|
|
202
|
+
if unique_op_info:
|
|
203
|
+
self.operation_info[op_name] = unique_op_info
|
|
204
|
+
else:
|
|
205
|
+
raise RuntimeError(f'The information of <{op_name}> is not found. Please verify that the operator name is'
|
|
206
|
+
f' correct or the operator is used in the network.')
|
|
207
|
+
|
|
208
|
+
def get_device_target_filename(self):
|
|
209
|
+
"""Get device target filename."""
|
|
210
|
+
gpu_framework_file = f'gpu_framework_{self._dev_id}.txt'
|
|
211
|
+
cpu_framework_file = f'cpu_framework_{self._dev_id}.txt'
|
|
212
|
+
gpu_activity_file = f'gpu_activity_data_{self._dev_id}.csv'
|
|
213
|
+
cpu_op_detail_file = f'cpu_op_detail_info_{self._dev_id}.csv'
|
|
214
|
+
all_file = os.listdir(self._output_path)
|
|
215
|
+
if not all_file:
|
|
216
|
+
raise RuntimeError(f'No profiler file is found in the path <%s>. '
|
|
217
|
+
f'Check whether the profiler path is correct.' % self._output_path)
|
|
218
|
+
if gpu_activity_file in all_file and gpu_framework_file not in all_file:
|
|
219
|
+
raise RuntimeError(f'The output file <%s> is not found.' % gpu_framework_file)
|
|
220
|
+
if cpu_op_detail_file in all_file and cpu_framework_file not in all_file:
|
|
221
|
+
raise RuntimeError(f'The output file <%s> is not found.' % cpu_framework_file)
|
|
222
|
+
if gpu_framework_file in all_file and gpu_activity_file not in all_file:
|
|
223
|
+
raise RuntimeError(f'The output file <%s> is not found.' % gpu_activity_file)
|
|
224
|
+
if cpu_framework_file in all_file and cpu_op_detail_file not in all_file:
|
|
225
|
+
raise RuntimeError(f'The output file <%s> is not found.' % cpu_op_detail_file)
|
|
226
|
+
if gpu_activity_file not in all_file and cpu_op_detail_file not in all_file:
|
|
227
|
+
raise RuntimeError(f'The profiling data of this card which device_id is equal to {self._dev_id} does not'
|
|
228
|
+
f' exist. Check whether device_id is correct.')
|
|
229
|
+
for file_name in all_file:
|
|
230
|
+
if file_name.endswith(f'activity_data_{self._dev_id}.csv'):
|
|
231
|
+
self.activity_info_dir.append(file_name)
|
|
232
|
+
if file_name.endswith(f'framework_{self._dev_id}.txt'):
|
|
233
|
+
self.framework_info_dir.append(file_name)
|
|
234
|
+
if file_name.startswith('cpu_op_detail') and file_name.endswith(f'{self._dev_id}.csv'):
|
|
235
|
+
self.cpu_detail_info_dir.append(file_name)
|
|
@@ -14,7 +14,10 @@
|
|
|
14
14
|
# ============================================================================
|
|
15
15
|
"""Profiler output path"""
|
|
16
16
|
import os
|
|
17
|
+
import glob
|
|
18
|
+
import re
|
|
17
19
|
from typing import Any, Dict, Optional
|
|
20
|
+
from mindspore import log as logger
|
|
18
21
|
from mindspore.profiler.common.path_manager import PathManager
|
|
19
22
|
|
|
20
23
|
|
|
@@ -49,12 +52,9 @@ class ProfilerOutputPath:
|
|
|
49
52
|
_MINISTUDIO_PROFILER_OUTPUT = "mindstudio_profiler_output"
|
|
50
53
|
_MINISTUDIO_ANALYZE_OUTPUT = "analyze"
|
|
51
54
|
|
|
52
|
-
def __init__(self, rank_id: int
|
|
53
|
-
if not isinstance(rank_id, int) or not isinstance(device_id, int):
|
|
54
|
-
raise ValueError("rank_id and device_id must be integers")
|
|
55
|
+
def __init__(self, rank_id: int):
|
|
55
56
|
|
|
56
57
|
self._rank_id = rank_id
|
|
57
|
-
self._device_id = device_id
|
|
58
58
|
self._output_path: Optional[str] = None
|
|
59
59
|
self._ascend_ms_dir: Optional[str] = None
|
|
60
60
|
self._ascend_profiler_output_path: Optional[str] = None
|
|
@@ -269,10 +269,25 @@ class ProfilerOutputPath:
|
|
|
269
269
|
self._msprof_profile_host_path = os.path.join(
|
|
270
270
|
self._msprof_profile_path, ProfilerOutputPath._MINISTUDIO_PROFILER_HOST
|
|
271
271
|
)
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
)
|
|
272
|
+
|
|
273
|
+
device_pattern = os.path.join(self._msprof_profile_path, "device_*")
|
|
274
|
+
device_dirs = glob.glob(device_pattern)
|
|
275
|
+
valid_device_dirs = [d for d in device_dirs if re.match(r'^device_\d+$', os.path.basename(d))]
|
|
276
|
+
|
|
277
|
+
if valid_device_dirs:
|
|
278
|
+
device_dir = os.path.basename(valid_device_dirs[0])
|
|
279
|
+
device_id = device_dir.replace("device_", "")
|
|
280
|
+
self._msprof_profile_device_path = os.path.join(
|
|
281
|
+
self._msprof_profile_path,
|
|
282
|
+
ProfilerOutputPath._MINISTUDIO_PROFILER_DEVICE.format(device_id)
|
|
283
|
+
)
|
|
284
|
+
else:
|
|
285
|
+
logger.error(f"No device_* directory found in {self._msprof_profile_path}, using device_0 as default")
|
|
286
|
+
self._msprof_profile_device_path = os.path.join(
|
|
287
|
+
self._msprof_profile_path,
|
|
288
|
+
ProfilerOutputPath._MINISTUDIO_PROFILER_DEVICE.format("0")
|
|
289
|
+
)
|
|
290
|
+
|
|
276
291
|
self._msprof_profile_log_path = os.path.join(
|
|
277
292
|
self._msprof_profile_path, ProfilerOutputPath._MINISTUDIO_PROFILER_LOG
|
|
278
293
|
)
|
|
@@ -21,7 +21,8 @@ from mindspore.profiler.common.constant import (
|
|
|
21
21
|
ProfilerLevel,
|
|
22
22
|
ProfilerActivity,
|
|
23
23
|
AicoreMetrics,
|
|
24
|
-
ExportType
|
|
24
|
+
ExportType,
|
|
25
|
+
HostSystem
|
|
25
26
|
)
|
|
26
27
|
from mindspore.profiler.schedule import Schedule
|
|
27
28
|
|
|
@@ -45,10 +46,16 @@ class ProfilerParameters:
|
|
|
45
46
|
"l2_cache": (bool, False),
|
|
46
47
|
"hbm_ddr": (bool, False),
|
|
47
48
|
"pcie": (bool, False),
|
|
49
|
+
"sys_io": (bool, False),
|
|
50
|
+
"sys_interconnection": (bool, False),
|
|
51
|
+
"host_sys": (list, []),
|
|
48
52
|
"sync_enable": (bool, True),
|
|
49
53
|
"data_simplification": (bool, True),
|
|
54
|
+
"record_shapes": (bool, False),
|
|
50
55
|
"export_type": (list, [ExportType.Text]),
|
|
51
56
|
"mstx": (bool, False),
|
|
57
|
+
"mstx_domain_include": (list, []),
|
|
58
|
+
"mstx_domain_exclude": (list, []),
|
|
52
59
|
"schedule": (Schedule, None),
|
|
53
60
|
"on_trace_ready": (Optional[Callable[..., Any]], None)
|
|
54
61
|
}
|
|
@@ -72,13 +79,9 @@ class ProfilerParameters:
|
|
|
72
79
|
params = {}
|
|
73
80
|
params["is_set_schedule"] = self.is_set_schedule
|
|
74
81
|
for param, (_, _) in self.PARAMS.items():
|
|
75
|
-
if param
|
|
82
|
+
if param in ["profiler_level", "aic_metrics"]:
|
|
76
83
|
params[param] = getattr(self, param).value
|
|
77
|
-
elif param
|
|
78
|
-
params[param] = getattr(self, param).value
|
|
79
|
-
elif param == "activities":
|
|
80
|
-
params[param] = [item.value for item in getattr(self, param)]
|
|
81
|
-
elif param == "export_type":
|
|
84
|
+
elif param in ["activities", "export_type", "host_sys"]:
|
|
82
85
|
params[param] = [item.value for item in getattr(self, param)]
|
|
83
86
|
elif param == "schedule":
|
|
84
87
|
params[param] = getattr(self, param).to_dict()
|
|
@@ -102,10 +105,16 @@ class ProfilerParameters:
|
|
|
102
105
|
"l2_cache": self.l2_cache,
|
|
103
106
|
"hbm_ddr": self.hbm_ddr,
|
|
104
107
|
"pcie": self.pcie,
|
|
108
|
+
"sys_io": self.sys_io,
|
|
109
|
+
"sys_interconnection": self.sys_interconnection,
|
|
110
|
+
"host_sys": ",".join([item.value for item in self.host_sys]) if self.host_sys else "",
|
|
105
111
|
"parallel_strategy": self.parallel_strategy,
|
|
106
112
|
"profiler_level": self.profiler_level.value,
|
|
107
113
|
"with_stack": self.with_stack,
|
|
114
|
+
"record_shapes": self.record_shapes,
|
|
108
115
|
"mstx": self.mstx,
|
|
116
|
+
"mstx_domain_include": self.mstx_domain_include,
|
|
117
|
+
"mstx_domain_exclude": self.mstx_domain_exclude,
|
|
109
118
|
"cpu_trace": ProfilerActivity.CPU in self.activities,
|
|
110
119
|
"npu_trace": ProfilerActivity.NPU in self.activities,
|
|
111
120
|
}
|
|
@@ -127,43 +136,62 @@ class ProfilerParameters:
|
|
|
127
136
|
if key in ProfilerParameters.PARAMS:
|
|
128
137
|
expected_type = ProfilerParameters.PARAMS[key][ProfilerParameters.TYPE_INDEX]
|
|
129
138
|
default_value = ProfilerParameters.PARAMS[key][ProfilerParameters.VALUE_INDEX]
|
|
130
|
-
|
|
131
|
-
# Callable特殊处理
|
|
132
139
|
if key == "on_trace_ready":
|
|
133
|
-
|
|
134
|
-
setattr(self, key, default_value)
|
|
135
|
-
logger.warning(
|
|
136
|
-
f"For Profiler, on_trace_ready value is Invalid, reset to {default_value}."
|
|
137
|
-
)
|
|
140
|
+
setattr(self, key, self._check_and_get_on_trace_ready(value, default_value))
|
|
138
141
|
elif key == "schedule":
|
|
139
|
-
|
|
140
|
-
setattr(self, key, Schedule(wait=0, active=1))
|
|
141
|
-
logger.warning(
|
|
142
|
-
f"For Profiler, schedule value is Invalid, reset to {Schedule(wait=0, active=1)}"
|
|
143
|
-
)
|
|
142
|
+
setattr(self, key, self._check_and_get_schedule(value))
|
|
144
143
|
elif key == "export_type":
|
|
145
144
|
setattr(self, key, self._check_and_get_export_type(value))
|
|
145
|
+
elif key in ("mstx_domain_include", "mstx_domain_exclude"):
|
|
146
|
+
setattr(self, key, self._check_and_get_mstx_domain(key, value))
|
|
147
|
+
elif key == "host_sys":
|
|
148
|
+
setattr(self, key, self._check_and_get_host_sys(value, expected_type, default_value))
|
|
146
149
|
# 检查可迭代类型
|
|
147
150
|
elif isinstance(expected_type, type) and issubclass(expected_type, (list, tuple, set)):
|
|
148
|
-
|
|
149
|
-
all(isinstance(item, type(default_value[0])) for item in value)):
|
|
150
|
-
logger.warning(
|
|
151
|
-
f"For Profiler, {key} value is Invalid, reset to {default_value}."
|
|
152
|
-
)
|
|
153
|
-
setattr(self, key, default_value)
|
|
151
|
+
setattr(self, key, self._check_and_get_iterable_params(key, value, expected_type, default_value))
|
|
154
152
|
# 检查普通类型
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
153
|
+
else:
|
|
154
|
+
setattr(self, key, self._check_and_get_common_params(key, value, expected_type, default_value))
|
|
155
|
+
|
|
156
|
+
@staticmethod
|
|
157
|
+
def _check_and_get_on_trace_ready(value, default_value):
|
|
158
|
+
if value is not None and not callable(value):
|
|
159
|
+
logger.warning(f"For Profiler, on_trace_ready value is Invalid, reset to {default_value}.")
|
|
160
|
+
return default_value
|
|
161
|
+
|
|
162
|
+
return value
|
|
163
|
+
|
|
164
|
+
@staticmethod
|
|
165
|
+
def _check_and_get_schedule(value):
|
|
166
|
+
if not isinstance(value, Schedule):
|
|
167
|
+
logger.warning(f"For Profiler, schedule value is Invalid, reset to {Schedule(wait=0, active=1)}")
|
|
168
|
+
return Schedule(wait=0, active=1)
|
|
169
|
+
|
|
170
|
+
return value
|
|
161
171
|
|
|
162
|
-
|
|
172
|
+
@staticmethod
|
|
173
|
+
def _check_and_get_iterable_params(key, value, expected_type, default_value):
|
|
174
|
+
if not (isinstance(value, expected_type) and all(isinstance(item, type(default_value[0])) for item in value)):
|
|
175
|
+
logger.warning(f"For Profiler, {key} value is Invalid, reset to {default_value}.")
|
|
176
|
+
return default_value
|
|
177
|
+
|
|
178
|
+
return list(set(value))
|
|
179
|
+
|
|
180
|
+
@staticmethod
|
|
181
|
+
def _check_and_get_common_params(key, value, expected_type, default_value):
|
|
182
|
+
if not isinstance(value, expected_type):
|
|
183
|
+
logger.warning(f"For Profiler, the type of {key} should be {expected_type}, "
|
|
184
|
+
f"but got {type(value)}, reset to {default_value}.")
|
|
185
|
+
return default_value
|
|
186
|
+
|
|
187
|
+
return value
|
|
188
|
+
|
|
189
|
+
@staticmethod
|
|
190
|
+
def _check_deprecated_params(**kwargs) -> None:
|
|
163
191
|
"""
|
|
164
192
|
Check deprecated parameters.
|
|
165
193
|
"""
|
|
166
|
-
for key,
|
|
194
|
+
for key, value in kwargs.items():
|
|
167
195
|
if key == "profile_communication":
|
|
168
196
|
warnings.warn(
|
|
169
197
|
"The parameter 'profile_communication' is deprecated,"
|
|
@@ -189,6 +217,16 @@ class ProfilerParameters:
|
|
|
189
217
|
warnings.warn(
|
|
190
218
|
"The parameter 'timeline_limit' is deprecated and will have no effect"
|
|
191
219
|
)
|
|
220
|
+
elif key == "pcie" and value is True:
|
|
221
|
+
warnings.warn(
|
|
222
|
+
"The parameter 'pcie' will be deprecated in future versions. "
|
|
223
|
+
"Please use 'sys_interconnection' in mindspore.profiler._ExperimentalConfig instead."
|
|
224
|
+
)
|
|
225
|
+
elif key == "hbm_ddr" and value is True:
|
|
226
|
+
warnings.warn(
|
|
227
|
+
"The parameter 'hbm_ddr' will be deprecated in future versions. "
|
|
228
|
+
"Please use 'profile_memory' in instead."
|
|
229
|
+
)
|
|
192
230
|
|
|
193
231
|
def _set_schedule(self, **kwargs):
|
|
194
232
|
if "schedule" in kwargs and isinstance(kwargs["schedule"], Schedule):
|
|
@@ -221,7 +259,29 @@ class ProfilerParameters:
|
|
|
221
259
|
f"'aic_metrics' cannot be set to 'AicoreMetrics.AiCoreNone', "
|
|
222
260
|
f"reset to 'AicoreMetrics.PipeUtilization'.")
|
|
223
261
|
|
|
224
|
-
|
|
262
|
+
if self.__dict__.get('record_shapes', False) and ProfilerActivity.CPU not in self.__dict__.get('activities'):
|
|
263
|
+
self.record_shapes = False
|
|
264
|
+
warnings.warn("when 'ProfilerActivity.CPU' is not set in 'activities', 'Record_shapes' cannot be set to "
|
|
265
|
+
"True, reset to 'False'.")
|
|
266
|
+
|
|
267
|
+
if not self.__dict__.get('mstx') and (self.__dict__.get('mstx_domain_include') or \
|
|
268
|
+
self.__dict__.get('mstx_domain_exclude')):
|
|
269
|
+
self.mstx_domain_include = []
|
|
270
|
+
self.mstx_domain_exclude = []
|
|
271
|
+
warnings.warn(
|
|
272
|
+
"when 'mstx' is set to 'False', 'mstx_domain_include' and 'mstx_domain_exclude' cannot be set, "
|
|
273
|
+
"reset them to '[]'."
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
if self.__dict__.get('mstx_domain_include') and self.__dict__.get('mstx_domain_exclude'):
|
|
277
|
+
self.mstx_domain_exclude = []
|
|
278
|
+
warnings.warn(
|
|
279
|
+
f"mstx_domain_include and mstx_domain_exclude can not be set together, "
|
|
280
|
+
f"mstx_domain_exclude has been reset to {self.mstx_domain_exclude}."
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
@staticmethod
|
|
284
|
+
def _check_and_get_export_type(export_type) -> list:
|
|
225
285
|
"""
|
|
226
286
|
Check export type.
|
|
227
287
|
"""
|
|
@@ -229,7 +289,7 @@ class ProfilerParameters:
|
|
|
229
289
|
return [ExportType.Text]
|
|
230
290
|
|
|
231
291
|
if isinstance(export_type, str):
|
|
232
|
-
if export_type in
|
|
292
|
+
if export_type in (ExportType.Text.value, ExportType.Db.value):
|
|
233
293
|
return [ExportType(export_type)]
|
|
234
294
|
|
|
235
295
|
if isinstance(export_type, list):
|
|
@@ -242,6 +302,39 @@ class ProfilerParameters:
|
|
|
242
302
|
logger.warning("Invalid parameter export_type, reset it to text.")
|
|
243
303
|
return [ExportType.Text]
|
|
244
304
|
|
|
305
|
+
@staticmethod
|
|
306
|
+
def _check_and_get_mstx_domain(list_name, domain_list) -> list:
|
|
307
|
+
"""
|
|
308
|
+
Check mstx domain.
|
|
309
|
+
"""
|
|
310
|
+
if not domain_list:
|
|
311
|
+
return []
|
|
312
|
+
if not isinstance(domain_list, list):
|
|
313
|
+
logger.warning(f"For Profiler, {list_name} value is Invalid, reset to [].")
|
|
314
|
+
return []
|
|
315
|
+
for domain_name in domain_list:
|
|
316
|
+
if not isinstance(domain_name, str) or domain_name == "":
|
|
317
|
+
logger.warning(f"{list_name} has value {domain_name} is not str or is empty, reset to [].")
|
|
318
|
+
return []
|
|
319
|
+
return list(set(domain_list))
|
|
320
|
+
|
|
321
|
+
@staticmethod
|
|
322
|
+
def _check_and_get_host_sys(host_sys, expected_type, default_value):
|
|
323
|
+
"""
|
|
324
|
+
Check host system.
|
|
325
|
+
"""
|
|
326
|
+
if not host_sys:
|
|
327
|
+
return default_value
|
|
328
|
+
|
|
329
|
+
if not (isinstance(host_sys, expected_type) and
|
|
330
|
+
all(isinstance(item, HostSystem) for item in host_sys)):
|
|
331
|
+
logger.warning(
|
|
332
|
+
f"For Profiler, 'host_sys' value is Invalid, reset to {default_value}."
|
|
333
|
+
)
|
|
334
|
+
return default_value
|
|
335
|
+
|
|
336
|
+
return set(host_sys)
|
|
337
|
+
|
|
245
338
|
def __getattr__(self, name):
|
|
246
339
|
"""
|
|
247
340
|
Get attribute.
|
|
File without changes
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# Copyright 2025-2026 Huawei Technologies Co., Ltd
|
|
2
|
+
#
|
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
|
+
# you may not use this file except in compliance with the License.
|
|
5
|
+
# You may obtain a copy of the License at
|
|
6
|
+
#
|
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
8
|
+
#
|
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
12
|
+
# See the License for the specific language governing permissions and
|
|
13
|
+
# limitations under the License.
|
|
14
|
+
# ============================================================================
|
|
15
|
+
"""Dynamic Profiler utils"""
|
|
16
|
+
from mindspore.profiler.common.singleton import Singleton
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@Singleton
|
|
20
|
+
class MsDynamicMonitorProxySingleton:
|
|
21
|
+
"""
|
|
22
|
+
Class for dyno monitor proxy.
|
|
23
|
+
"""
|
|
24
|
+
def __init__(self):
|
|
25
|
+
self._proxy = None
|
|
26
|
+
self._load_success = True
|
|
27
|
+
|
|
28
|
+
def _load_proxy(self):
|
|
29
|
+
if not self._proxy and self._load_success:
|
|
30
|
+
try:
|
|
31
|
+
from IPCMonitor import PyDynamicMonitorProxy
|
|
32
|
+
except ImportError:
|
|
33
|
+
self._load_success = False
|
|
34
|
+
return
|
|
35
|
+
self._proxy = PyDynamicMonitorProxy()
|
|
36
|
+
|
|
37
|
+
def get_proxy(self):
|
|
38
|
+
self._load_proxy()
|
|
39
|
+
return self._proxy
|