mindstudio-probe 8.3.0__py3-none-any.whl → 8.3.2__py3-none-any.whl

msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py

@@ -1,378 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from functools import partial
-import zlib
-import io
-import struct
-import time
-import os
-from queue import Queue
-from threading import Thread
-from typing import Union
-
-from twisted.internet import reactor, protocol, endpoints, ssl
-from twisted.protocols.basic import FileSender
-
-from msprobe.pytorch.common.utils import logger
-from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.utils import STRUCT_UNPACK_MODE as unpack_mode, \
-    STR_TO_BYTES_ORDER as bytes_order, cipher_list, verify_callback, load_ssl_pem
-
-MAX_SENDING_QUEUE_SIZE = 20
-
-
-class TCPDataItem:
-    def __init__(self, data,
-                 sequence_number: int,
-                 rank: int = 0,
-                 step: int = 0):
-        self.raw_data = data
-        self.sequence_number = sequence_number
-        self.rank = rank
-        self.step = step
-        self.retry_times = 0
-        self.pending_time = 0
-        self.busy_time = 0
-
-
-class TCPClient:
-    ACK_SUCCESS = b"OK___"
-    ACK_ERROR = b"ERROR"
-    ACK_BUSY = b"BUSY_"
-    ACK_STOP = b"STOP_"
-    ACK_STOP_CONFIRM = b"OVER_"
-    ACK_KILL_PROCESS = b"KILL_"
-
-    QUEUE_PENDING_TIME = 60
-    RESEND_RETRY_TIMES = 2  # 最大重传数
-    RESEND_TIMER_TIME = 5  # 接收ACK超时定时器
-    RESEND_PENDING_TIME = 60  # 连续pending时间超过1分钟则放弃该数据
-
-    def __init__(self, host="localhost", port=8000, check_sum=False, tls_path=None):
-        self.send_queue = Queue(MAX_SENDING_QUEUE_SIZE)
-        self.resend_dict = dict()
-        self.host = host
-        self.port = port
-        self.tls_path = tls_path
-        self.factory = None
-        self.sequence_number = 0
-        self.signal_exit = False
-        self.tcp_manager = ClientProtocol(ack_queue_size=100,
-                                          chunk_size=655360,
-                                          check_sum=check_sum,
-                                          tls=self.tls_path)
-        self.send_thread = Thread(target=self._sending_queue_data)
-        self.send_thread.setDaemon(True)
-        self.send_thread.start()
-        self.destroy_thread = Thread(target=self._destroy_queue_data)
-        self.destroy_thread.setDaemon(True)
-        self.destroy_thread.start()
-
-    @staticmethod
-    def run_reactor():
-        reactor.run(installSignalHandlers=False)
-
-    def start(self):
-        def conn_callback(cur_protocol):
-            if cur_protocol.transport and cur_protocol.transport.getPeer().host == self.host:
-                logger.debug(f"Process: {os.getpid()} connects to server successfully.")
-            else:
-                logger.warning(f"Process: {os.getpid()} fails to connect to server. ")
-                raise ConnectionError(f"Failed to connect to {self.host}.")
-
-        def conn_err_callback(failure):
-            self.signal_exit = True
-            time.sleep(1)
-            reactor.stop()
-            logger.error(f"Failed to connected {self.host} {self.port}. Reason is {failure.getErrorMessage()}")
-
-        def cur_protocol():
-            return self.tcp_manager
-
-        self.factory = MessageClientFactory()
-        self.factory.protocol = cur_protocol
-        if self.tls_path:
-            client_key, client_crt, ca_crt, crl_pem = load_ssl_pem(
-                key_file=os.path.join(self.tls_path, "client.key"),
-                cert_file=os.path.join(self.tls_path, "client.crt"),
-                ca_file=os.path.join(self.tls_path, "ca.crt"),
-                crl_file=os.path.join(self.tls_path, "crl.pem")
-            )
-
-            ssl_options = ssl.CertificateOptions(
-                privateKey=client_key,
-                certificate=client_crt,
-                method=ssl.SSL.TLSv1_2_METHOD,
-                verify=True,
-                requireCertificate=True,
-                caCerts=[ca_crt],  # 信任的CA证书列表
-            )
-            ssl_context = ssl_options.getContext()
-            ssl_context.set_cipher_list(cipher_list)
-            ssl_context.set_options(ssl.SSL.OP_NO_RENEGOTIATION)
-            ssl_context.set_verify(ssl.SSL.VERIFY_PEER | ssl.SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
-                                   partial(verify_callback, crl=crl_pem))
-
-            endpoint = endpoints.SSL4ClientEndpoint(reactor, self.host, self.port, ssl_options)
-        else:
-            endpoint = endpoints.TCP4ClientEndpoint(reactor, self.host, self.port)
-        d = endpoint.connect(self.factory)
-        d.addCallback(conn_callback)
-        d.addErrback(conn_err_callback)
-
-        reactor_thread = Thread(target=self.run_reactor, daemon=True)
-        reactor_thread.start()
-
-    def send_after_queue_empty(self, data):
-        while not self._ready_to_exit():
-            if not self.tls_path:
-                self.add_to_sending_queue(data)
-            else:
-                for _ in range(MAX_SENDING_QUEUE_SIZE):
-                    self.add_to_sending_queue(data)
-            time.sleep(2)
-
-    def check_client_alive(self):
-        return self.factory.num_connections > 0
-
-    def stop(self):
-        self.tcp_manager.connection_timeout()
-
-    def send_stop_signal(self):
-        self.send_after_queue_empty(self.ACK_STOP)
-        while not self._ready_to_exit():
-            if not self.check_client_alive():
-                break
-            time.sleep(1)
-
-    def add_to_sending_queue(self, data: Union[bytes, TCPDataItem], rank: int = 0, step: int = 0):
-        if self._ready_to_exit():
-            return
-
-        send_data = data
-        if not isinstance(data, TCPDataItem):
-            send_data = TCPDataItem(data=data,
-                                    sequence_number=self.sequence_number,
-                                    rank=rank,
-                                    step=step)
-            self.sequence_number += 1
-        try:
-            self.send_queue.put(send_data, block=True, timeout=self.QUEUE_PENDING_TIME)
-        except Exception as e:
-            logger.error(f"send_queue put send_data timeout, rank: {send_data.rank}, step: {send_data.step},"
-                         f"sequence_number: {send_data.sequence_number}, send_queue size: {self.send_queue.qsize()},"
-                         f"{str(e)}")
-
-    def _send_data(self, data: TCPDataItem):
-        self.tcp_manager.send_wrapped_data(data.raw_data,
-                                           sequence_number=data.sequence_number,
-                                           rank=data.rank,
-                                           step=data.step
-                                           )
-
-    def _sending_queue_data(self):
-        while True:
-            if not self.tcp_manager.is_connected:
-                continue
-
-            while self.send_queue.qsize() > 0:
-                if self._ready_to_exit():
-                    break
-                if len(self.resend_dict) < MAX_SENDING_QUEUE_SIZE:
-                    data_obj = self.send_queue.get()
-                    resend_key = str(data_obj.sequence_number) + "_" + str(data_obj.rank) + "_" + str(data_obj.step)
-                    logger.debug(f"get {resend_key} from send_queue, and send to server.")
-                    self._send_data(data_obj)
-                    if resend_key not in self.resend_dict.keys():
-                        # Send data for the first time
-                        self.resend_dict[resend_key] = data_obj
-                else:
-                    time.sleep(0.1)
-
-            if self._ready_to_exit():
-                logger.debug("Successfully close sending process.")
-                break
-            time.sleep(0.1)
-
-    def _destroy_queue_data(self):
-        while True:
-            if self._ready_to_exit():
-                break
-
-            while len(self.resend_dict) > 0 and self.tcp_manager.ack_queue.qsize() > 0:
-                ack_info, seq_number, rank, step = self.tcp_manager.ack_queue.get()
-                obj_key = str(seq_number) + "_" + str(rank) + "_" + str(step)
-                current_item = self.resend_dict.get(obj_key)
-
-                if current_item is None:
-                    continue
-
-                if ack_info == self.ACK_SUCCESS:
-                    self.resend_dict.pop(obj_key)
-                elif ack_info == self.ACK_BUSY:
-                    logger.debug("RECV BUSY ACK")
-                    if current_item.busy_time > 5:
-                        self._resend_data(current_item)
-                    else:
-                        current_item.busy_time += 1
-                elif ack_info == self.ACK_ERROR:
-                    logger.debug("RECV ERROR ACK")
-                    self._resend_data(current_item)
-                elif ack_info == self.ACK_STOP_CONFIRM:
-                    logger.debug("RECV STOP ACK")
-                    self.factory.num_connections -= 1
-
-                break
-
-            time.sleep(0.1)
-
-    def _resend_data(self, data: TCPDataItem):
-        if data.retry_times < self.RESEND_RETRY_TIMES:
-            data.retry_times += 1
-            logger.debug(f"Resend data seq number: {data.sequence_number}")
-            self.add_to_sending_queue(data)
-        else:
-            self.resend_dict.pop(data.sequence_number)
-            logger.debug(f"SKIP send sequence number {data.sequence_number} after retry {data.retry_times} times!")
-
-    def _pending_data(self, data: TCPDataItem):
-        if data.pending_time >= self.RESEND_PENDING_TIME:
-            self.resend_dict.pop(data.sequence_number)
-            logger.debug(f"SKIP send sequence number {data.sequence_number} after pending {data.pending_time} times!")
-            return
-
-        # wait time is 100MB per second
-        pending_time = max(1, len(data.raw_data) // (2 ** 20 * 50))
-        data.pending_time += pending_time
-        time.sleep(pending_time)
-
-    def _ready_to_exit(self):
-        return self.signal_exit or self.tcp_manager.signal_exit
-
-
-class ClientProtocol(protocol.Protocol):
-    TIMEOUT = 60 * 10
-
-    def __init__(self, ack_queue_size=100, chunk_size=65536, check_sum=False, tls=None):
-        self.buffer = io.BytesIO()
-        self.is_connected = False
-        self.check_sum = check_sum
-        self.tell = 0
-        self.ack_queue = Queue(maxsize=ack_queue_size)
-        self.file_sender = FileSender()
-        self.file_sender.CHUNK_SIZE = chunk_size
-        self.signal_exit = False
-        self.defer = None
-        self.kill_process = False
-        self.ack = None
-
-        self.timeout_call = None
-
-        self.tls = tls
-        self.send_buffer = b""
-        self.buffer_cnt = 0
-
-    def dataReceived(self, data):
-        if self.timeout_call.active():
-            self.timeout_call.reset(self.TIMEOUT)
-
-        self.buffer.seek(0, 2)
-        self.buffer.write(data)
-        self.buffer.seek(self.tell)
-        while True:
-            if len(self.buffer.getvalue()) >= 29:  # 5 + 8 * 3
-                ack = self.buffer.read(5)
-                self.ack = ack
-                seq_number = struct.unpack(unpack_mode, self.buffer.read(8))[0]
-                rank = struct.unpack(unpack_mode, self.buffer.read(8))[0]
-                step = struct.unpack(unpack_mode, self.buffer.read(8))[0]
-                logger.debug(f"receive 流水号: {seq_number}; RANK: {rank}; STEP: {step}; ACK: {ack}")
-                if ack == b"KILL_":
-                    self.kill_process = True
-                    logger.debug(f"接收到KILL信号, PID {os.getpid()}")
-                if ack == b"OVER_":
-                    self.factory.num_connections -= 1
-                self.tell += 29
-                if not self.ack_queue.full():
-                    self.ack_queue.put((ack, seq_number, rank, step))
-                    self.buffer = io.BytesIO(self.buffer.getvalue()[self.tell:])
-                    self.tell = 0
-                else:
-                    time.sleep(0.1)
-            else:
-                break
-
-    def send_wrapped_data(self, data, sequence_number: int = 0, rank: int = 0, step: int = 0):
-        length = len(data)
-        data_crc = f"{zlib.crc32(data):08x}" if self.check_sum else ""
-        data_meaasge = length.to_bytes(8, byteorder=bytes_order) + \
-                       sequence_number.to_bytes(8, byteorder=bytes_order) + \
-                       rank.to_bytes(8, byteorder=bytes_order) + \
-                       step.to_bytes(8, byteorder=bytes_order) + \
-                       data_crc.encode() + \
-                       data
-        logger.debug(f"send 流水号: {sequence_number}; RANK: {rank}; STEP: {step}; LENGTH: {length}")
-
-        while True:
-            if self.defer is None or self.defer.called:
-                self.defer = self.send_large_data(data_meaasge)
-                break
-            time.sleep(0.01)
-
-    def send_large_data(self, data):
-
-        if self.tls:
-            self.send_buffer += data
-            self.buffer_cnt += 1
-            if self.buffer_cnt >= MAX_SENDING_QUEUE_SIZE:
-                d = self.file_sender.beginFileTransfer(io.BytesIO(self.send_buffer), self.transport)
-                self.send_buffer = b""
-                self.buffer_cnt = 0
-            else:
-                d = None
-        else:
-            d = self.file_sender.beginFileTransfer(io.BytesIO(data), self.transport)
-        return d
-
-    def connection_timeout(self):
-        if self.factory.num_connections <= 0:
-            return
-
-        self.factory.num_connections -= 1
-        logger.debug(f"超时退出{self.transport.addr}, PID {os.getpid()}")
-        self.transport.loseConnection()
-
-    def connectionMade(self):
-        self.timeout_call = reactor.callLater(self.TIMEOUT, self.connection_timeout)
-        self.is_connected = True
-        self.factory.num_connections += 1
-        logger.info("successfully connect server")
-
-    def connectionLost(self, reason):
-        self.signal_exit = True
-        self.factory.num_connections -= 1
-        logger.info(f"Lost connection with server, reason is : {reason.value}")
-
-
-class MessageClientFactory(protocol.ClientFactory):
-    def __init__(self):
-        self.num_connections = 0
-
-    def clientConnectionFailed(self, connector, reason):
-        logger.info(f"Fail to connection with server: {reason.getErrorMessage()}")
-        reactor.stop()
-
-    def clientConnectionLost(self, connector, reason):
-        logger.info(f"Client lost connection with server: {reason.getErrorMessage()}")
-        reactor.stop()

msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py

@@ -1,239 +0,0 @@
-# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd.
-# All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0  (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import time
-from collections import namedtuple
-
-import pandas as pd
-import torch
-import torch.multiprocessing as mp
-
-from msprobe.core.common.const import Const, CompareConst
-from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import online_api_precision_compare
-from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import DETAIL_TEST_ROWS, thousandth_standard_api, \
-    binary_standard_api, absolute_standard_api
-from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import UtDataInfo, exec_api, ExecParams
-from msprobe.pytorch.common.log import logger
-from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import move2target_device
-from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import generate_cpu_params
-
-# NPU vs GPU api list
-CompareApi = set(absolute_standard_api) | set(binary_standard_api) | set(thousandth_standard_api)
-
-current_time = time.strftime("%Y%m%d%H%M%S")
-ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + "_rank*.csv"
-ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + "_rank*.csv"
-
-OnlineApiPrecisionCompareConfig = namedtuple('OnlineApiPrecisionCompareConfig',
-                                             ['npu_data', 'gpu_data', 'rank', 'result_csv_path', 'details_csv_path'])
-# namedtuple of [instance of Comparator, func of run_touch_api_online, config of run_ut_config]
-CommonCompareConfig = namedtuple('CommonCompareConfig', ['compare', 'handle_func', 'config'])
-
-
-def get_gpu_device():
-    is_gpu = False
-    try:
-        import torch_npu
-    except ImportError:
-        is_gpu = True
-    return is_gpu
-
-
-def run_ut_process(xpu_id, consumer_queue, common_config, api_precision_csv_file):
-    """ When consumer_queue(shared with ConsumerDispatcher) is not empty, consume api data from consumer_queue.
-    :param xpu_id: int
-    :param consumer_queue: shared queues of ConsumerDispatcher
-    :param common_config: namedtuple of CommonCompareConfig
-    :param api_precision_csv_file: list, length is 2, result file name and details file name
-    :return:
-    """
-    device_info = "cuda" if get_gpu_device() else "npu"
-    logger.info(f"Start run_ut_process for {device_info} device, rank: {xpu_id}.")
-    gpu_device = torch.device(f'{device_info}:{xpu_id}')
-
-    while True:
-        if consumer_queue.empty():
-            time.sleep(0.1)
-            continue
-
-        api_data = consumer_queue.get()
-        if api_data == "KILL_":
-            # current consumer finish
-            return
-
-        _, api_name, _ = api_data.name.split(Const.SEP)
-        if api_name in CompareApi:
-            # NPU vs GPU
-            online_compare(api_data, gpu_device, common_config)
-        else:
-            # NPUvsCPU vs GPUvsCPU
-            online_precision_compare(api_data, gpu_device, common_config, api_precision_csv_file)
-
-
-def online_precision_compare(api_data, device, common_config, api_precision_csv_file):
-    """online run_ut for precision_compare: NPUvsCPU vs GPUvsCPU
-    1. get NPUvsCPU compare result
-    2. get GPUvsCPU compare result
-    3. call online_api_precision_compare
-    :param api_data
-    :param device
-    :param common_config: namedtuple of CommonCompareConfig
-    :param api_precision_csv_file: [result_file_name, details_file_name]
-    """
-    compare, func, config = common_config.compare, common_config.handle_func, common_config.config
-    api_full_name = api_data.name
-    [api_type, api_name, _] = api_full_name.split(Const.SEP)
-    npu_args, npu_kwargs, npu_out = api_data.args, api_data.kwargs, api_data.result
-
-    if npu_kwargs.get("device"):
-        del npu_kwargs["device"]
-
-    try:
-        # NPU vs CPU
-        cpu_params = generate_cpu_params(npu_args, npu_kwargs, False, api_name)
-        cpu_args, cpu_kwargs = cpu_params.cpu_args, cpu_params.cpu_kwargs
-        cpu_exec_params = ExecParams(api_type, api_name, Const.CPU_LOWERCASE, cpu_args, cpu_kwargs, False, None)
-        cpu_out = exec_api(cpu_exec_params)
-        npu_data_info = UtDataInfo(None, None, npu_out, cpu_out, None, [], None, rank=api_data.rank)
-        npu_detail = compare.compare_output(api_full_name, npu_data_info, True)
-        npu_data = pd.DataFrame(npu_detail, columns=DETAIL_TEST_ROWS[-1])
-
-        # GPU vs CPU
-        api_data_gpu = move2target_device(api_data, device)  # args, kwargs -> gpu, result -> npu
-        data_info = func(api_full_name, api_data_gpu, config.backward_content)
-        gpu_out = data_info.bench_output
-        gpu_data_info = UtDataInfo(None, None, gpu_out, cpu_out, None, [], None, rank=api_data.rank)
-        gpu_detail = compare.compare_output(api_full_name, gpu_data_info, True)
-        gpu_data = pd.DataFrame(gpu_detail, columns=DETAIL_TEST_ROWS[-1])
-
-        # NPUvsCPU vs GPUvsCPU
-        result_file_name, details_file_name = api_precision_csv_file
-        precision_compare_config = OnlineApiPrecisionCompareConfig(npu_data, gpu_data, api_data.rank,
-                                                                   result_file_name, details_file_name)
-        online_api_precision_compare(precision_compare_config)
-
-    except Exception as err:
-        if "expected scalar type Long" in str(err):
-            logger.warning(
-                f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API "
-                f"'int32_to_int64' list in accuracy_tools/msprobe/core/common/const.py file.")
-        elif api_type in [Const.DISTRIBUTED]:
-            logger.info(f"{api_full_name} is not supported for run ut. SKIP.")
-        else:
-            logger.error(f"Run {api_full_name} UT Error: {str(err)}")
-
-        compare.write_summary_csv((api_full_name, CompareConst.SKIP, CompareConst.SKIP, [[str(err)]], api_data.rank))
-
-    finally:
-        torch.cuda.empty_cache()
-
-
-def online_compare(api_data, device, common_config):
-    """online run_ut for compare：NPU vs GPU
-    """
-    compare, func, config = common_config.compare, common_config.handle_func, common_config.config
-    api_full_name = api_data.name
-    api_data = move2target_device(api_data, device)
-    try:
-        data_info = func(api_full_name, api_data, config.backward_content)
-        is_fwd_success, is_bwd_success = compare.compare_output(api_full_name, data_info)
-        logger.info(f"running api_full_name {api_full_name} ut, "
-                    f"is_fwd_success: {is_fwd_success}, "
-                    f"is_bwd_success: {is_bwd_success}")
-    except Exception as err:
-        [api_type, api_name, _] = api_full_name.split(Const.SEP)
-        if "expected scalar type Long" in str(err):
-            logger.warning(
-                f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API "
-                f"'int32_to_int64' list in accuracy_tools/msprobe/core/common/const.py file.")
-        elif api_type in [Const.DISTRIBUTED]:
-            logger.info(f"{api_full_name} is not supported for run ut. SKIP.")
-        else:
-            logger.error(f"Run {api_full_name} UT Error: {str(err)}")
-
-        compare.write_summary_csv((api_full_name, CompareConst.SKIP, CompareConst.SKIP, [[str(err)]], api_data.rank))
-
-    finally:
-        torch.cuda.empty_cache()
-
-
-class ConsumerDispatcher:
-    def __init__(self, compare, capacity=10, num_workers=8, device: str = "gpu") -> None:
-        self.num_workers = num_workers
-        self.capacity = capacity
-        self.compare = compare
-        self.queues = []
-        self.processes = []
-        self.reverse_sort = False
-        self.pool = None
-        self.device = device
-        self.data_id = 0
-        self.lock = mp.Lock()
-        self.result_queue = mp.Queue()
-        mp.set_start_method("spawn", force=True)
-
-    def start(self, handle_func, config):
-        self.queues = [mp.Queue(maxsize=self.capacity) for _ in range(self.num_workers)]
-        api_precision_csv_file = [
-            ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME,
-            ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME
-        ]
-        common_config = CommonCompareConfig(self.compare, handle_func, config)
-        for xpu_id, q in enumerate(self.queues):
-            p = mp.Process(name="run_ut_process", target=run_ut_process,
-                           args=(xpu_id, q, common_config, api_precision_csv_file))
-
-            p.start()
-            self.processes.append(p)
-        logger.info(
-            f'Api_precision_compare task result will be saved in {ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME}')
-        logger.info(
-            f"Api_precision_compare task details will be saved in {ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME}")
-        logger.info("Successfully start unittest process.")
-
-    def stop(self):
-        for q in self.queues:
-            while q.full():
-                time.sleep(0.1)
-            q.put("KILL_")
-
-        for p in self.processes:
-            p.join()
-        logger.info("Successfully stop unittest process.")
-        logger.info(f"Api_precision_compare task result is saved in {ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME}")
-        logger.info(f"Api_precision_compare task details is saved in {ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME}")
-
-    def update_consume_queue(self, api_data):
-        while True:
-            index = self._choose_max_empty_site_strategy()
-            if index != -1:
-                q = self.queues[index]
-                q.put(api_data)
-                break
-            time.sleep(0.1)
-
-    def _choose_max_empty_site_strategy(self):
-        maximum = 0
-        index = -1
-        # 充分利用多卡资源，防止任务过多分配给前面的卡
-        _reverse = 1 if not self.reverse_sort else -1
-        for i, q in enumerate(self.queues[::_reverse]):
-            empty_site = self.capacity - q.qsize()
-            if empty_site > maximum:
-                maximum = empty_site
-                index = i
-        index = len(self.queues) - index - 1 if index != -1 and self.reverse_sort else index
-        self.reverse_sort = not self.reverse_sort
-        return index