mindspore 2.4.0__cp310-cp310-macosx_10_15_x86_64.whl

mindspore/profiler/parser/ascend_fpbp_generator.py

@@ -0,0 +1,82 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""fp bp and is_training_mode_flag analyse model"""
+import json
+import logging
+import os
+import stat
+from collections import defaultdict
+
+import numpy as np
+from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException
+
+
+class AscendFPBPGenerator:
+    """Generate ascend fp bp data from DataFrame."""
+
+    def __init__(self, op_summary, steptrace, pretty=False):
+        self.op_summary = op_summary
+        self.steptrace = steptrace
+        self.points = None
+        self.pretty = pretty
+
+    @property
+    def indent(self):
+        indent = 1 if self.pretty else None
+        return indent
+
+    def parse(self):
+        """Analyse the op_summary and steptrace data generate fpbp data."""
+        is_training_mode_flag = False
+        points = None
+
+        steptrace = self.steptrace[self.steptrace['Iteration ID'] == 1]
+        steptrace = steptrace[(steptrace['FP Start'] != 0) & (steptrace['BP End'] != 0)]
+        if steptrace.size:
+            is_training_mode_flag = True
+            op_summary = self.op_summary[np.argsort(self.op_summary['Task Start Time'])]
+            fp_index = np.searchsorted(op_summary['Task Start Time'], steptrace['FP Start'], side='right') - 1
+            bp_index = np.searchsorted(op_summary['Task Start Time'] + op_summary['Task Duration'], steptrace['BP End'],
+                                       side='left') - 1
+            points = defaultdict(dict)
+            for i in range(steptrace.size):
+                model_id = f"model_{steptrace[i]['Iteration ID']}"
+                points[model_id]['fp_start'] = op_summary[fp_index[i]]['Op Name']
+                points[model_id]['bp_end'] = op_summary[bp_index[i]]['Op Name']
+            self.points = defaultdict()
+            self.points['fp_start'] = op_summary[fp_index[0]]['Op Name']
+            self.points['bp_end'] = op_summary[bp_index[0]]['Op Name']
+
+        return points, is_training_mode_flag
+
+    def write(self, step_trace_point_info_path):
+        """
+        Write the flops.csv and flops_summary.json
+
+        Args:
+            step_trace_point_info_path(str): step_trace_point_info.json path.
+        """
+        if not self.points:
+            return
+        try:
+            with os.fdopen(os.open(step_trace_point_info_path,
+                                   os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR),
+                           'w') as json_file:
+                json.dump(self.points, json_file, indent=self.indent)
+        except (IOError, OSError) as err:
+            logging.critical('Errot occurred when write step trace point info file: %s', err)
+            raise ProfilerIOException() from err
+        if os.path.exists(step_trace_point_info_path):
+            os.chmod(step_trace_point_info_path, stat.S_IREAD | stat.S_IWRITE)

mindspore/profiler/parser/ascend_hccl_generator.py

@@ -0,0 +1,271 @@
+# Copyright 2023 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""hccl analyse model"""
+import copy
+import csv
+import json
+import logging
+import os
+import stat
+import glob
+
+import numpy as np
+from mindspore.profiler.common.exceptions.exceptions import ProfilerIOException
+from mindspore.profiler.common.util import get_newest_file
+
+
+class AscendHCCLGenerator:
+    """Generate ascend hccl data from files."""
+
+    DEFAULT_MODEL_ID = 4294967295
+
+    def __init__(self, mindstudio_profiler_output, steptrace):
+        self.mindstudio_profiler_output = mindstudio_profiler_output
+        self.steptrace = steptrace
+        self.hccl_raw = []
+        self.hccl_data_df = np.dtype(
+            [('model_id', int), ('iteration_id', int), ('name', object), ('pid', int), ('tid', int), ('ts', float),
+             ('te', float), ('dur', float), ('ph', object),
+             ('task_type', object), ('link_info', object), ('transport_type', object), ('size', int), ('tag', object)])
+
+    @staticmethod
+    def _cost_analyse(iteration):
+        """analyse communication cost and wait cost"""
+        communication_cost = np.sum(iteration[iteration['name'] != 'Notify_Wait']['dur'])
+        wait_cost = np.sum(iteration[iteration['name'] == 'Notify_Wait']['dur'])
+        return communication_cost, wait_cost
+
+    @staticmethod
+    def _rdma_analyse(groupby_transport):
+        """rdma analyse"""
+        thread_groups = np.unique(groupby_transport['tid'])
+        thread_information = []
+        for thread_index in thread_groups:
+            groupby_thread = groupby_transport[groupby_transport['tid'] == thread_index]
+            rdma_communication_time = 0
+            rdma_communication_size = 0
+            rdma_communication_wait_time = 0
+            start_index = 0
+            end_index = groupby_thread.size - 2
+            while start_index < end_index:
+                first_task_type = groupby_thread[start_index]['task_type']
+                if first_task_type == 'RDMASend':
+                    second_index = start_index + 1
+                    third_index = start_index + 2
+                    second_task_type = groupby_thread[second_index]['task_type']
+                    third_task_type = groupby_thread[third_index]['task_type']
+                    if second_task_type == 'RDMASend' and third_task_type == 'Notify Wait':
+                        rdma_send_cost = groupby_thread[start_index]['dur']
+                        notify_record_cost = groupby_thread[second_index]['dur']
+                        notify_wait_cost = groupby_thread[third_index]['dur']
+                        rdma_communication_time += rdma_send_cost + notify_record_cost + notify_wait_cost
+                        rdma_communication_wait_time += notify_wait_cost
+                        rdma_communication_size += groupby_thread[start_index]['size'] + groupby_thread[second_index][
+                            'size']
+                        start_index += 2
+                start_index += 1
+            rdma_communication_wait_time = rdma_communication_wait_time / 1e3
+            rdma_communication_size = rdma_communication_size / 1e3
+            rdma_communication_time = rdma_communication_time / 1e3
+            rdma_bandwidth = rdma_communication_size / (rdma_communication_time / 1e3) \
+                if rdma_communication_size else 0
+            thread_information.append(
+                [rdma_communication_time, rdma_communication_size, rdma_bandwidth, rdma_communication_wait_time])
+        if len(thread_information) > 1:
+            thread_information = np.sum(thread_information, axis=0).tolist()
+
+        return thread_information
+
+    def parse(self):
+        """Analyse the original hccl data generator hccl data."""
+
+        raw_data = []
+        msprof_name = fr'{self.mindstudio_profiler_output}/msprof_*.json'
+        for msprof_file in get_newest_file(glob.glob(msprof_name)):
+            with open(msprof_file) as fr:
+                raw_data.extend(json.load(fr))
+
+        hccl_data = self._original_data_analyse(raw_data)
+
+        for model_id in np.unique(hccl_data['model_id']):
+            hccl_data_model = hccl_data[hccl_data['model_id'] == model_id]
+            for iteration_id in np.unique(hccl_data_model['iteration_id']):
+                hccl_data_model_iteration = hccl_data_model[hccl_data_model['iteration_id'] == iteration_id]
+
+                hccl_abstract_data = hccl_data_model_iteration[hccl_data_model_iteration['task_type'] == '']
+                hccl_detail_data = hccl_data_model_iteration[hccl_data_model_iteration['task_type'] != '']
+                hccl_abstract_data = np.sort(hccl_abstract_data, order='ts')
+                hccl_detail_data = np.sort(hccl_detail_data, order='ts')
+
+                tag = np.searchsorted(hccl_abstract_data['ts'], hccl_detail_data['ts'], side='right') - 1
+
+                hccl_detail_data['tag'] = [x[-1] for x in
+                                           np.char.split(hccl_abstract_data[tag]['name'].astype(str), sep='/')]
+
+                self.hccl_raw.append(self._iteration_analyse(hccl_detail_data, iteration_id))
+
+        self.hccl_raw = sorted(self.hccl_raw, key=lambda x: x[0])
+        if self.hccl_raw:
+            self.hccl_raw.append(copy.deepcopy(self.hccl_raw[-1]))
+            self.hccl_raw[-1][0] = '-'
+            for _, value in self.hccl_raw[-1][4].items():
+                value[0] = '-'
+
+    def write(self, hccl_raw_path):
+        """
+        Write the flops.csv and flops_summary.json
+
+        Args:
+            hccl_raw_path(str): hccl_raw.csv path.
+        """
+        try:
+            with os.fdopen(os.open(hccl_raw_path,
+                                   os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR), 'w',
+                           newline='') as hccl_row:
+                writer = csv.writer(hccl_row)
+                writer.writerow(
+                    ['step_num', 'communication_cost', 'wait_cost', 'link_info', 'communication_operator_cost'])
+                for row in self.hccl_raw:
+                    row[3] = json.dumps(row[3])
+                    row[4] = json.dumps(row[4])
+                writer.writerows(self.hccl_raw)
+        except (IOError, OSError) as err:
+            logging.critical('Errot occurred when write aicore detail file: %s', err)
+            raise ProfilerIOException() from err
+        if os.path.exists(hccl_raw_path):
+            os.chmod(hccl_raw_path, stat.S_IREAD | stat.S_IWRITE)
+
+    def _original_data_analyse(self, original_data):
+        """analyse original data"""
+
+        groups_steptrace = {model_id: np.sort(self.steptrace[self.steptrace['Model ID'] == model_id],
+                                              order='Iteration ID')
+                            for model_id in np.unique(self.steptrace['Model ID'])}
+
+        hccl_pid = None
+        for row in original_data:
+            if row.get('ph') == 'M' and row.get('name') == 'process_name' and row.get('args', {}).get('name') == 'HCCL':
+                hccl_pid = row.get('pid')
+                break
+
+        target_data = []
+        for row in original_data:
+            model_id = row.get('args', {}).get('model id')
+            pid = row.get('pid')
+            if hccl_pid == pid and row.get('ph') == 'X' and model_id is not None:
+                name = row.get('name')
+                tid = row.get('tid')
+                ts = float(row.get('ts'))
+                dur = float(row.get('dur'))
+                te = ts + dur
+                ph = row.get('ph')
+                task_type = row.get('args', {}).get('task type', '')
+                src_rank = row.get('args', {}).get('src rank', 0)
+                dst_rank = row.get('args', {}).get('dst rank', 0)
+                if src_rank == int('0xffffffff', 16):
+                    src_rank = dst_rank
+                if dst_rank == int('0xffffffff', 16):
+                    dst_rank = src_rank
+                transport_type = row.get('args', {}).get('transport type', '')
+                if transport_type == 'LOCAL':
+                    src_rank, dst_rank = dst_rank, src_rank
+                link_info = str(src_rank) + '-' + str(dst_rank)
+                size = row.get('args', {}).get('size(Byte)', 0)
+                size = size if isinstance(size, int) else int(size, 16)
+                steptrace = groups_steptrace.get(model_id, None)
+                if steptrace is None:
+                    if self.DEFAULT_MODEL_ID != model_id:
+                        logging.warning('Could not find model: %s in hccl json, skip.', model_id)
+                    continue
+                tag = np.searchsorted(steptrace['Iteration End'], te * 1e-3, side='left')
+                iteration_id = steptrace[tag]['Iteration ID']
+                target_data.append(
+                    tuple([model_id, iteration_id, name, pid, tid,
+                           ts, te, dur, ph, task_type,
+                           link_info, transport_type, size, -1]))
+
+        hccl_data = np.array(target_data, dtype=self.hccl_data_df)
+
+        return hccl_data
+
+    def _iteration_analyse(self, hccl_detail_data, iteration):
+        """analyse data by iteration """
+        communication_cost, wait_cost = self._cost_analyse(hccl_detail_data)
+        link_info = self._link_info_analyse(hccl_detail_data)
+        communication_operator_cost = self._communication_operator_cost_analyse(hccl_detail_data, iteration)
+        return [iteration, communication_cost, wait_cost, link_info, communication_operator_cost]
+
+    def _link_info_analyse(self, hccl_detail_data):
+        """analyse link info data"""
+        groupby_iteration = hccl_detail_data[hccl_detail_data['task_type'] != 'Notify Record']
+        link_info_groups = np.unique(groupby_iteration['link_info'])
+        link_info_information = dict()
+        for link_info_index in link_info_groups:
+            groupby_link_info = groupby_iteration[groupby_iteration['link_info'] == link_info_index]
+            transport_groups = np.unique(groupby_iteration['transport_type'])
+            transport_information = dict()
+            for transport_index in transport_groups:
+                groupby_transport = groupby_link_info[groupby_link_info['transport_type'] == transport_index]
+                if transport_index == 'SDMA' and groupby_transport.size > 0:
+                    groupby_sdma = \
+                        groupby_transport[np.isin(groupby_transport['task_type'], ['Memcpy', 'Reduce Inline'])][
+                            ['dur', 'size']]
+                    sdma_communication_time = np.sum(groupby_sdma['dur']) * 1e-3
+                    sdma_communication_size = np.sum(groupby_sdma['size']) * 1e-3
+                    sdma_bandwidth = sdma_communication_size / sdma_communication_time * 1e-3 \
+                        if sdma_communication_time != 0 else 0
+                    transport_information['SDMA'] = [sdma_communication_time, sdma_communication_size, sdma_bandwidth]
+                elif transport_index == 'RDMA' and groupby_transport.size > 0:
+                    transport_information['RDMA'] = self._rdma_analyse(groupby_transport)
+            link_info_information[link_info_index] = transport_information
+        return link_info_information
+
+    def _communication_operator_cost_analyse(self, hccl_detail_data, iteration_index):
+        """analyse communication operator cost"""
+        groupby_iteration = hccl_detail_data[hccl_detail_data['task_type'] != 'Notify Record']
+        tag_groups = np.unique(groupby_iteration['tag'])
+        tag_information = dict()
+        for tag_index in tag_groups:
+            groupby_tag = groupby_iteration[groupby_iteration['tag'] == tag_index]
+            link_groups = np.unique(groupby_iteration['link_info'])
+            link_info_information = dict()
+            for link_info_index in link_groups:
+                groupby_link_info = groupby_tag[groupby_tag['link_info'] == link_info_index]
+                transport_groups = np.unique(groupby_link_info['transport_type'])
+                transport_information = dict()
+                for transport_index in transport_groups:
+                    groupby_transport = groupby_link_info[groupby_link_info['transport_type'] == transport_index]
+                    if transport_index == 'SDMA':
+                        groupby_sdma = \
+                            groupby_transport[np.isin(groupby_transport['task_type'], ['Memcpy', 'Reduce Inline'])][
+                                ['dur', 'size']]
+                        sdma_communication_time = np.sum(groupby_sdma['dur']) * 1e-3
+                        sdma_communication_size = np.sum(groupby_sdma['size']) * 1e-3
+                        sdma_bandwidth = sdma_communication_size / sdma_communication_time * 1e-3 \
+                            if sdma_communication_time != 0 else 0
+                        transport_information['SDMA'] = [
+                            sdma_communication_time, sdma_communication_size,
+                            sdma_bandwidth
+                        ]
+                    elif transport_index == 'RDMA':
+                        transport_information['RDMA'] = self._rdma_analyse(groupby_transport)
+                    link_info_information[link_info_index] = transport_information
+                communication_cost = np.sum(groupby_tag[groupby_tag['name'] != 'Notify_Wait']['dur'])
+                wait_cost = np.sum(groupby_tag[groupby_tag['name'] == 'Notify_Wait']['dur'])
+                tag_information[tag_index] = [
+                    str(iteration_index), communication_cost, wait_cost,
+                    link_info_information
+                ]
+        return tag_information

mindspore/profiler/parser/ascend_integrate_generator.py

@@ -0,0 +1,42 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""PROF csv data analyze module"""
+import os
+
+from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
+
+
+class AscendIntegrateGenerator:
+    """Generate ms profiler output csv"""
+
+    def __init__(self, mindstudio_profiler_output: str, ascend_ms_output_path: str):
+        self._mindstudio_profiler_output = mindstudio_profiler_output
+        self._ascend_ms_output_path = ascend_ms_output_path
+
+    def parse(self):
+        """Generate ms profiler output csv"""
+        self._generate_kernel_details()
+
+    def _generate_kernel_details(self):
+        """Generate kernel_details.csv"""
+        header_map = {
+            "Op Name": "Name", "OP Type": "Type", "Task Type": "Accelerator Core",
+            "Task Start Time(us)": "Start Time(us)", "Task Duration(us)": "Duration(us)",
+            "Task Wait Time(us)": "Wait Time(us)",
+        }
+        op_summary_file_list = FileManager.get_csv_file_list_by_start_name(self._mindstudio_profiler_output,
+                                                                           "op_summary")
+        kernel_details_file = os.path.join(self._ascend_ms_output_path, "kernel_details.csv")
+        FileManager.combine_csv_file(op_summary_file_list, kernel_details_file, header_map)

mindspore/profiler/parser/ascend_memory_generator.py

@@ -0,0 +1,185 @@
+# Copyright 2024 Huawei Technologies Co., Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ============================================================================
+"""memory data analyze module"""
+import os
+
+from decimal import Decimal
+
+from mindspore.profiler.parser.ascend_analysis.file_manager import FileManager
+
+
+class AscendMemoryGenerator:
+    """Parsing the memory data of the ascend device"""
+    GE_MEMORY_RECORD_HEADERS = ["Device id", "Component", "Timestamp(us)", "Total Allocated(KB)",
+                                "Total Reserved(KB)", "Device"]
+    MS_MEMORY_RECORD_HEADERS = ["Timestamp(ns)", "Total Allocated(Byte)", "Total Reserved(Byte)", "Total Active(Byte)"]
+    TARGET_MEMORY_RECORD_HEADERS = ["Component", "Timestamp(us)", "Total Allocated(MB)", "Total Reserved(MB)",
+                                    "Total Active(MB)", "Device Type"]
+
+    def __init__(self, source_path: str, rank_id: int, mindstudio_profiler_output: str, ascend_ms_output_path: str):
+        self._source_path = source_path
+        self._rank_id = rank_id
+        self._mindstudio_profiler_output = mindstudio_profiler_output
+        self._ascend_ms_output_path = ascend_ms_output_path
+        self._ge_memory_record = []
+        self._ms_memory_record = []
+
+    def parse(self):
+        """Parse all memory data"""
+        self.copy_npu_module_mem_csv()
+        self.parse_memory_record()
+
+    def copy_npu_module_mem_csv(self):
+        """Generate npu_module_mem.csv"""
+        npu_module_mem_file_list = FileManager.get_csv_file_list_by_start_name(self._mindstudio_profiler_output,
+                                                                               "npu_module_mem")
+        target_file_path = os.path.join(self._ascend_ms_output_path, "npu_module_mem.csv")
+        FileManager.combine_csv_file(npu_module_mem_file_list, target_file_path)
+
+    def parse_memory_record(self):
+        """Generate memory_record.csv"""
+        self._parse_ge_memory_record()
+        self._parse_ms_memory_record()
+        combined_memory_data = self._combine_ge_ms_memory_record()
+        target_file_path = os.path.join(self._ascend_ms_output_path, "memory_record.csv")
+        FileManager.create_csv_file(target_file_path, combined_memory_data, self.TARGET_MEMORY_RECORD_HEADERS)
+
+    def _parse_ge_memory_record(self):
+        """Parse ge memory record data"""
+        memory_record_file_list = FileManager.get_csv_file_list_by_start_name(self._mindstudio_profiler_output,
+                                                                              "memory_record")
+        for file in memory_record_file_list:
+            data = FileManager.read_csv_file(file)
+            if len(data) > 1:
+                self._ge_memory_record.extend(data[1:])
+
+    def _parse_ms_memory_record(self):
+        """Parse mindspore memory record data"""
+        memory_record_file = os.path.join(self._source_path, f"cpu_ms_memory_record_{self._rank_id}.txt")
+        data = FileManager.read_txt_file(memory_record_file)
+        if len(data) > 1:
+            self._ms_memory_record.extend(data[1:])
+
+    def _get_app_reserved_memory(self) -> list:
+        """Get the reserved memory of the application from npu_mem.csv"""
+        npu_module_mem_file_list = FileManager.get_csv_file_list_by_start_name(self._mindstudio_profiler_output,
+                                                                               "npu_mem")
+        app_mems = []
+        for file in npu_module_mem_file_list:
+            md_mems = FileManager.read_csv_file(file)
+            for mem in md_mems:
+                if mem[1] == "APP":
+                    app_mems.append(MemoryRecordBean([mem[1], mem[-1].rstrip('\t'), 0.0, float(mem[4]), 0.0,
+                                                      f"NPU:{self._rank_id}"]).row)
+
+        return app_mems
+
+    def _combine_ge_ms_memory_record(self) -> list:
+        """Combine ge and mindspore memory record data"""
+        memory_records = []
+        for ge_memory in self._ge_memory_record:
+            memory_record = dict(zip(self.GE_MEMORY_RECORD_HEADERS, ge_memory))
+            memory_records.append(
+                MemoryRecordBean([memory_record.get("Component", "GE"), memory_record.get("Timestamp(us)"),
+                                  memory_record.get("Total Allocated(KB)", 0),
+                                  memory_record.get("Total Reserved(KB)", 0),
+                                  0, memory_record.get("Device")]))
+        for ms_memory in self._ms_memory_record:
+            memory_record = dict(zip(self.MS_MEMORY_RECORD_HEADERS, ms_memory))
+            memory_records.append(
+                MemoryRecordBean(["MindSpore", Decimal(memory_record.get("Timestamp(ns)", 0)) / 1000,
+                                  float(memory_record.get("Total Allocated(Byte)", 0)) / 1024,
+                                  float(memory_record.get("Total Reserved(Byte)", 0)) / 1024,
+                                  float(memory_record.get("Total Active(Byte)", 0)) / 1024,
+                                  f"NPU:{self._rank_id}"]))
+        memory_records.sort(key=lambda x: x.time_us)
+        last_ge_memory, last_ms_memory = MemoryRecordBean([0] * 6), MemoryRecordBean([0] * 6)
+        result_data = []
+        for memory_record in memory_records:
+            result_data.append(memory_record.row)
+            last_memory = last_ms_memory if memory_record.is_ge_component() else last_ge_memory
+            combined_mem = MemoryRecordBean(["MindSpore+GE", memory_record.time_us,
+                                             memory_record.total_allocated_kb + last_memory.total_allocated_kb,
+                                             memory_record.total_reserved_kb + last_memory.total_reserved_kb,
+                                             memory_record.total_active_kb + last_memory.total_active_kb,
+                                             f"NPU:{self._rank_id}"])
+            result_data.append(combined_mem.row)
+            if memory_record.is_ge_component():
+                last_ge_memory = memory_record
+            else:
+                last_ms_memory = memory_record
+        return result_data + self._get_app_reserved_memory()
+
+
+class MemoryRecordBean:
+    """Memory Record Struct"""
+    KB_TO_MB = 1000
+    HEADERS = ["Component", "Timestamp(us)", "Total Allocated(KB)", "Total Reserved(KB)",
+               "Total Active(KB)", "Device Type"]
+
+    def __init__(self, data: list):
+        self._data = dict(zip(self.HEADERS, data))
+
+    @property
+    def row(self) -> list:
+        return [self.component, self.time_us_str, self.total_allocated_mb, self.total_reserved_mb,
+                self.total_active_mb, self.device_type]
+
+    @property
+    def component(self) -> str:
+        return self._data.get("Component", "")
+
+    @property
+    def time_us_str(self) -> str:
+        ts_us = self._data.get("Timestamp(us)", 0)
+        return str(ts_us)
+
+    @property
+    def total_allocated_mb(self) -> float:
+        return float(self._data.get("Total Allocated(KB)", 0)) / self.KB_TO_MB
+
+    @property
+    def total_reserved_mb(self) -> float:
+        return float(self._data.get("Total Reserved(KB)", 0)) / self.KB_TO_MB
+
+    @property
+    def total_active_mb(self) -> float:
+        return float(self._data.get("Total Active(KB)", 0)) / self.KB_TO_MB
+
+    @property
+    def device_type(self) -> float:
+        return self._data.get("Device Type", "")
+
+    @property
+    def total_allocated_kb(self) -> float:
+        return float(self._data.get("Total Allocated(KB)", 0))
+
+    @property
+    def total_reserved_kb(self) -> float:
+        return float(self._data.get("Total Reserved(KB)", 0))
+
+    @property
+    def total_active_kb(self) -> float:
+        return float(self._data.get("Total Active(KB)", 0))
+
+    @property
+    def time_us(self) -> Decimal:
+        return Decimal(self._data.get("Timestamp(us)", 0))
+
+    def is_ge_component(self):
+        """
+        Determine if it is GE
+        """
+        return self.component == "GE"