evalscope 0.6.1__py3-none-any.whl → 0.7.1__py3-none-any.whl

evalscope/perf/utils/db_util.py

@@ -0,0 +1,200 @@
+import base64
+import os
+import pickle
+import sqlite3
+import sys
+from datetime import datetime
+
+import json
+from tabulate import tabulate
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.utils.benchmark_util import BenchmarkData, BenchmarkMetrics
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def encode_data(data) -> str:
+    """Encodes data using base64 and pickle."""
+    return base64.b64encode(pickle.dumps(data)).decode('utf-8')
+
+
+def write_json_file(data, output_path):
+    with open(output_path, 'w') as f:
+        json.dump(data, f, indent=4, ensure_ascii=False)
+
+
+def transpose_results(data):
+    headers = data.keys()
+    rows = zip(*data.values())
+
+    return [dict(zip(headers, row)) for row in rows]
+
+
+def create_result_table(cursor):
+    cursor.execute('''CREATE TABLE IF NOT EXISTS result(
+                      request TEXT,
+                      start_time REAL,
+                      chunk_times TEXT,
+                      success INTEGER,
+                      response_messages TEXT,
+                      completed_time REAL,
+                      latency REAL,
+                      first_chunk_latency REAL,
+                      n_chunks INTEGER,
+                      chunk_time REAL,
+                      prompt_tokens INTEGER,
+                      completion_tokens INTEGER,
+                      max_gpu_memory_cost REAL)''')
+
+
+def insert_benchmark_data(cursor: sqlite3.Cursor, benchmark_data: BenchmarkData):
+    request = encode_data(benchmark_data.request)
+    chunk_times = json.dumps(benchmark_data.chunk_times)
+    response_messages = encode_data(benchmark_data.response_messages)
+
+    # Columns common to both success and failure cases
+    common_columns = (
+        request,
+        benchmark_data.start_time,
+        chunk_times,
+        benchmark_data.success,
+        response_messages,
+        benchmark_data.completed_time,
+    )
+
+    if benchmark_data.success:
+        # Add additional columns for success case
+        additional_columns = (
+            benchmark_data.query_latency,
+            benchmark_data.first_chunk_latency,
+            benchmark_data.n_chunks,
+            benchmark_data.n_chunks_time,
+            benchmark_data.prompt_tokens,
+            benchmark_data.completion_tokens,
+            benchmark_data.max_gpu_memory_cost,
+        )
+        query = """INSERT INTO result(
+                      request, start_time, chunk_times, success, response_messages,
+                      completed_time, latency, first_chunk_latency,
+                      n_chunks, chunk_time, prompt_tokens, completion_tokens, max_gpu_memory_cost
+                   ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"""
+        cursor.execute(query, common_columns + additional_columns)
+    else:
+        query = """INSERT INTO result(
+                      request, start_time, chunk_times, success, response_messages, completed_time
+                   ) VALUES (?, ?, ?, ?, ?, ?)"""
+        cursor.execute(query, common_columns)
+
+
+def get_result_db_path(name, model):
+    current_time = datetime.now().strftime('%Y%m%d_%H%M%S')
+    output_dir = './outputs'
+    result_db_path = os.path.join(output_dir, f'{name or model}_perf', current_time, 'benchmark_data.db')
+
+    if not os.path.exists(os.path.dirname(result_db_path)):
+        os.makedirs(os.path.dirname(result_db_path), exist_ok=True)
+
+    logger.info(f'Save the result to: {result_db_path}')
+    if os.path.exists(result_db_path):
+        logger.warning('The db file exists, delete it and start again!.')
+        sys.exit(1)
+
+    return result_db_path
+
+
+def get_percentile_results(result_db_path: str):
+
+    def percentile_results(rows, index, percentiles):
+        results = {}
+        n_success_queries = len(rows)
+        for percentile in percentiles:
+            idx = int(n_success_queries * percentile / 100)
+            row = rows[idx]
+            value = row[index] if row[index] is not None else float('inf')
+            results[percentile] = round(value, 4)
+        return results
+
+    query_sql = ('SELECT start_time, chunk_times, success, completed_time, latency, first_chunk_latency, '
+                 'n_chunks, chunk_time, prompt_tokens, completion_tokens '
+                 'FROM result WHERE success=1 ORDER BY first_chunk_latency ASC')
+    percentiles = [10, 25, 50, 66, 75, 80, 90, 95, 98, 99]
+
+    with sqlite3.connect(result_db_path) as con:
+        rows = con.execute(query_sql).fetchall()
+
+    if len(rows) <= len(percentiles):
+        logger.info('Too little data to calculate quantiles!')
+        return {}
+
+    # Calculate percentiles for first chunk latency and latency
+    first_chunk_latency_index = 5
+    latency_index = 4
+
+    first_chunk_latency_results = percentile_results(rows, first_chunk_latency_index, percentiles)
+    rows.sort(key=lambda x: x[latency_index])
+    latency_results = percentile_results(rows, latency_index, percentiles)
+
+    # Prepare data for tabulation
+    return {
+        'Percentile': [f'{p}%' for p in percentiles],
+        'First Chunk Latency (s)': [first_chunk_latency_results[p] for p in percentiles],
+        'Latency (s)': [latency_results[p] for p in percentiles]
+    }
+
+
+def summary_result(args: Arguments, metrics: BenchmarkMetrics, expected_number_of_queries: int, result_db_path: str):
+    result_path = os.path.dirname(result_db_path)
+    write_json_file(args.to_dict(), os.path.join(result_path, 'benchmark_args.json'))
+
+    data = metrics.create_message()
+    data.update({'Expected number of requests': expected_number_of_queries, 'Result DB path': result_db_path})
+    write_json_file(data, os.path.join(result_path, 'benchmark_summary.json'))
+
+    # Print summary in a table
+    table = tabulate(list(data.items()), headers=['Key', 'Value'], tablefmt='grid')
+    logger.info('\nBenchmarking summary:\n' + table)
+
+    # Get percentile results
+    percentile_result = get_percentile_results(result_db_path)
+    if percentile_result:
+        write_json_file(transpose_results(percentile_result), os.path.join(result_path, 'benchmark_percentile.json'))
+        # Print percentile results in a table
+        table = tabulate(percentile_result, headers='keys', tablefmt='pretty')
+        logger.info('\nPercentile results:\n' + table)
+
+    if args.dataset.startswith('speed_benchmark'):
+        speed_benchmark_result(result_db_path)
+
+
+def speed_benchmark_result(result_db_path: str):
+    query_sql = """
+        SELECT
+            prompt_tokens,
+            ROUND(AVG(completion_tokens / latency), 2) AS avg_completion_token_per_second,
+            ROUND(AVG(max_gpu_memory_cost), 2)
+        FROM
+            result
+        WHERE
+            success = 1 AND latency > 0
+        GROUP BY
+            prompt_tokens
+    """
+
+    with sqlite3.connect(result_db_path) as con:
+        cursor = con.cursor()
+        cursor.execute(query_sql)
+        rows = cursor.fetchall()
+
+    # Prepare data for tabulation
+    headers = ['Prompt Tokens', 'Speed(tokens/s)', 'GPU Memory(GB)']
+    data = [dict(zip(headers, row)) for row in rows]
+
+    # Print results in a table
+    table = tabulate(data, headers='keys', tablefmt='pretty')
+    logger.info('\nSpeed Benchmark Results:\n' + table)
+
+    # Write results to JSON file
+    result_path = os.path.dirname(result_db_path)
+    write_json_file(data, os.path.join(result_path, 'speed_benchmark.json'))

evalscope/perf/utils/handler.py

@@ -0,0 +1,46 @@
+import asyncio
+import functools
+import signal
+import sys
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def exception_handler(func):
+    if asyncio.iscoroutinefunction(func):
+
+        @functools.wraps(func)
+        async def async_wrapper(*args, **kwargs):
+            try:
+                return await func(*args, **kwargs)
+            except Exception as e:
+                logger.exception(f"Exception in async function '{func.__name__}': {e}")
+                sys.exit(1)
+
+        return async_wrapper
+    else:
+
+        @functools.wraps(func)
+        def sync_wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            except Exception as e:
+                logger.exception(f"Exception in function '{func.__name__}': {e}")
+                sys.exit(1)
+
+        return sync_wrapper
+
+
+def signal_handler(signal_name, loop):
+    logger.info('Got signal %s: exit' % signal_name)
+    loop.stop()
+
+
+def add_signal_handlers(loop):
+    for signal_name in {'SIGINT', 'SIGTERM'}:
+        loop.add_signal_handler(
+            getattr(signal, signal_name),
+            functools.partial(signal_handler, signal_name, loop),
+        )

evalscope/perf/utils/local_server.py

@@ -0,0 +1,139 @@
+import os
+import subprocess
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+
+import torch
+import uvicorn
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from sse_starlette.sse import EventSourceResponse
+
+from evalscope.perf.arguments import Arguments
+from evalscope.perf.utils.chat_service import ChatCompletionRequest, ChatService, ModelList, TextCompletionRequest
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@dataclass
+class ServerSentEvent(object):
+
+    def __init__(self, data='', event=None, id=None, retry=None):
+        self.data = data
+        self.event = event
+        self.id = id
+        self.retry = retry
+
+    @classmethod
+    def decode(cls, line):
+        """Decode line to ServerSentEvent
+
+
+        Args:
+            line (str): The line.
+
+        Return:
+            ServerSentEvent (obj:`ServerSentEvent`): The ServerSentEvent object.
+
+        """
+        if not line:
+            return None
+        sse_msg = cls()
+        # format data:xxx
+        field_type, _, field_value = line.partition(':')
+        if field_value.startswith(' '):  # compatible with openai api
+            field_value = field_value[1:]
+        if field_type == 'event':
+            sse_msg.event = field_value
+        elif field_type == 'data':
+            field_value = field_value.rstrip()
+            sse_msg.data = field_value
+        elif field_type == 'id':
+            sse_msg.id = field_value
+        elif field_type == 'retry':
+            sse_msg.retry = field_value
+        else:
+            pass
+
+        return sse_msg
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    yield
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+
+
+def create_app(args) -> FastAPI:
+    app = FastAPI(lifespan=lifespan)
+    chat_service = ChatService(model_path=args.model, attn_implementation=args.attn_implementation)
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=['*'],
+        allow_credentials=True,
+        allow_methods=['*'],
+        allow_headers=['*'],
+    )
+
+    @app.get('/v1/models', response_model=ModelList)
+    async def list_models():
+        return await chat_service.list_models()
+
+    @app.post('/v1/completions')
+    async def create_text_completion(request: TextCompletionRequest):
+        return await chat_service._text_completion(request)
+
+    @app.post('/v1/chat/completions')
+    async def create_chat_completion(request: ChatCompletionRequest):
+        if request.stream:
+            return EventSourceResponse(chat_service._stream_chat(request))
+        else:
+            return await chat_service._chat(request)
+
+    return app
+
+
+def start_app(args: Arguments):
+    if args.api == 'local':
+        app = create_app(args)
+        uvicorn.run(app, host='0.0.0.0', port=8877, workers=1)
+
+    elif args.api == 'local_vllm':
+        os.environ['VLLM_USE_MODELSCOPE'] = 'True'
+
+        proc = subprocess.Popen([
+            'python', '-m', 'vllm.entrypoints.openai.api_server', '--model', args.model, '--served-model-name',
+            os.path.basename(args.model), '--tensor-parallel-size',
+            str(torch.cuda.device_count()), '--max-model-len', '32768', '--gpu-memory-utilization', '0.9', '--host',
+            '0.0.0.0', '--port', '8877', '--disable-log-requests', '--disable-log-stats'
+        ])
+        import atexit
+
+        def on_exit():
+            if proc.poll() is None:
+                logger.info('Terminating the child process...')
+                proc.terminate()
+                try:
+                    proc.wait(timeout=10)
+                except subprocess.TimeoutExpired:
+                    logger.warning('Child process did not terminate within the timeout, killing it forcefully...')
+                    proc.kill()
+                    proc.wait()
+                logger.info('Child process terminated.')
+            else:
+                logger.info('Child process has already terminated.')
+
+        atexit.register(on_exit)
+    else:
+        raise ValueError(f'Unknown API type: {args.api}')
+
+
+if __name__ == '__main__':
+    from collections import namedtuple
+
+    args = namedtuple('Args', ['model', 'attn_implementation', 'api'])
+
+    start_app(args(model='Qwen/Qwen2.5-0.5B-Instruct', attn_implementation=None, api='local_vllm'))

evalscope/registry/config/cfg_arena.yaml

@@ -0,0 +1,77 @@
+# input raw data
+question_file: registry/data/question.jsonl
+
+# candidate models to be battled
+answers_gen:
+    chatglm3-6b:
+        # model_id_or_path could be local absolute path, e.g. /to/path/.cache/modelscope/ZhipuAI/chatglm3-6b
+        model_id_or_path: ZhipuAI/chatglm3-6b       # model_id on modelscope
+        revision: v1.0.2        # revision of model, default is NULL
+        precision: torch.float16
+        enable: true            # enable or disable this model
+        template_type: chatglm3  # see: https://github.com/modelscope/swift/blob/main/docs/source/LLM/%E6%94%AF%E6%8C%81%E7%9A%84%E6%A8%A1%E5%9E%8B%E5%92%8C%E6%95%B0%E6%8D%AE%E9%9B%86.md
+        generation_config:
+            do_sample: true
+            max_new_tokens: 256
+            top_k: 20
+            top_p: 0.75
+            temperature: 0.333
+        # output predicted answer file name
+        output_file: registry/data/arena/answers/answer_chatglm3-6b.jsonl
+    Baichuan2-7B-Base:
+        model_id_or_path: baichuan-inc/Baichuan2-7B-Base
+        revision: v1.0.2       # revision of model, default is NULL
+        precision: torch.float16
+        enable: false           # enable or disable this model
+        template_type: default-generation
+        generation_config:
+            do_sample: true
+            max_new_tokens: 256
+            top_k: 20
+            top_p: 0.75
+            temperature: 0.3
+        output_file: registry/data/arena/answers/answer_Baichuan2-7B-Base.jsonl
+    Qwen-7B:
+        model_id_or_path: qwen/Qwen-7B
+        revision: v1.1.8       # revision of model, default is NULL
+        precision: torch.float16
+        enable: true           # enable or disable this model      # TODO: tokenizer issue
+        template_type: default-generation
+        generation_config:
+            do_sample: true
+            max_new_tokens: 256
+            top_k: 20
+            top_p: 0.75
+            temperature: 0.3
+        output_file: registry/data/arena/answers/answer_Qwen-7B.jsonl
+
+# Auto-reviewer(GPT-4) config
+reviews_gen:
+    enable: true
+    reviewer:
+        # class reference of auto reviewer(GPT-4)
+        ref: evalscope.evaluator.reviewer.auto_reviewer:AutoReviewerGpt4
+        args:
+            max_tokens: 1024
+            temperature: 0.2
+            # options: pairwise, pairwise_baseline, single (default is pairwise)
+            mode: pairwise
+            # position bias mitigation strategy, options: swap_position, randomize_order, NULL. default is NULL
+            position_bias_mitigation: NULL
+            # completion parser config, default is lmsys_parser
+            fn_completion_parser: lmsys_parser
+    # prompt templates for auto reviewer(GPT-4)
+    prompt_file: registry/data/prompt_template/prompt_templates.jsonl
+    # target answer files list to be reviewed,
+    # could be replaced by your own path: ['/path/to/answers_model_1.jsonl', '/path/to/answers_model_2.jsonl', ...]
+    # Default is NULL, which means all answers in answers_gen will be reviewed
+    target_answers: NULL
+    # output file name of auto reviewer
+    review_file: registry/data/arena/reviews/review_gpt4.jsonl
+
+# rating results
+rating_gen:
+    enable: true
+    metrics: ['elo']
+    # elo rating report file name
+    report_file: registry/data/arena/reports/elo_rating_origin.csv

evalscope/registry/config/cfg_arena_zhihu.yaml

@@ -0,0 +1,63 @@
+# input raw data
+question_file: registry/data/question.jsonl
+
+# candidate models to be battled
+answers_gen:
+    Qwen2-7B-Instruct:
+        model_id_or_path: /mnt/data/data/user/maoyunlin.myl/models/Qwen2-7B-Instruct       # model_id on modelscope
+        revision: NULL        # revision of model, default is NULL
+        precision: torch.float16
+        enable: true            # enable or disable this model
+        template_type: default-generation  # see: https://github.com/modelscope/swift/blob/main/docs/source/LLM/%E6%94%AF%E6%8C%81%E7%9A%84%E6%A8%A1%E5%9E%8B%E5%92%8C%E6%95%B0%E6%8D%AE%E9%9B%86.md
+        generation_config:
+            do_sample: true
+            max_new_tokens: 512
+            top_k: 20
+            top_p: 0.9
+            temperature: 0.7
+        # output predicted answer file name
+        output_file: registry/data/arena/answers/answer_qwen2.jsonl
+    Qwen-7B:
+        model_id_or_path: /mnt/data/data/user/maoyunlin.myl/output/qwen2-7b-instruct/v25-20240809-113533/checkpoint-309-merged
+        revision: NULL       # revision of model, default is NULL
+        precision: torch.float16
+        enable: true           # enable or disable this model     
+        template_type: default-generation
+        generation_config:
+            do_sample: true
+            max_new_tokens: 512
+            top_k: 20
+            top_p: 0.9
+            temperature: 0.7
+        output_file: registry/data/arena/answers/answer_Qwen-7B.jsonl
+
+# Auto-reviewer(GPT-4) config
+reviews_gen:
+    enable: true
+    reviewer:
+        # class reference of auto reviewer(GPT-4)
+        ref: evalscope.evaluator.reviewer.auto_reviewer:AutoReviewerGpt4
+        args:
+            max_tokens: 1024
+            temperature: 0.2
+            # options: pairwise, pairwise_baseline, single (default is pairwise)
+            mode: pairwise
+            # position bias mitigation strategy, options: swap_position, randomize_order, NULL. default is NULL
+            position_bias_mitigation: NULL
+            # completion parser config, default is lmsys_parser
+            fn_completion_parser: lmsys_parser
+    # prompt templates for auto reviewer(GPT-4)
+    prompt_file: registry/data/prompt_template/prompt_templates.jsonl
+    # target answer files list to be reviewed,
+    # could be replaced by your own path: ['/path/to/answers_model_1.jsonl', '/path/to/answers_model_2.jsonl', ...]
+    # Default is NULL, which means all answers in answers_gen will be reviewed
+    target_answers: NULL
+    # output file name of auto reviewer
+    review_file: registry/data/arena/reviews/review_gpt4.jsonl
+
+# rating results
+rating_gen:
+    enable: true
+    metrics: ['elo']
+    # elo rating report file name
+    report_file: registry/data/arena/reports/elo_rating_origin.csv

evalscope/registry/config/cfg_pairwise_baseline.yaml

@@ -0,0 +1,83 @@
+# input raw data
+question_file: registry/data/question.jsonl
+
+# candidate models to be battled
+answers_gen:
+    chatglm3-6b:
+        # model_id_or_path could be local absolute path, e.g. /to/path/.cache/modelscope/ZhipuAI/chatglm3-6b
+        model_id_or_path: ZhipuAI/chatglm3-6b       # model_id on modelscope
+        revision: v1.0.2        # revision of model, default is NULL
+        precision: torch.float16
+        enable: true            # enable or disable this model
+        template_type: chatglm3  # see: https://github.com/modelscope/swift/blob/main/docs/source/LLM/%E6%94%AF%E6%8C%81%E7%9A%84%E6%A8%A1%E5%9E%8B%E5%92%8C%E6%95%B0%E6%8D%AE%E9%9B%86.md
+        generation_config:
+            do_sample: true
+            max_new_tokens: 256
+            top_k: 20
+            top_p: 0.75
+            temperature: 0.3
+        # output predicted answer file name
+        output_file: registry/data/arena/answers/answer_chatglm3-6b.jsonl
+    Baichuan2-7B-Base:
+        model_id_or_path: baichuan-inc/Baichuan2-7B-Base
+        revision: v1.0.2       # revision of model, default is NULL
+        precision: torch.float16
+        enable: false           # enable or disable this model
+        template_type: default-generation
+        generation_config:
+            do_sample: true
+            max_new_tokens: 256
+            top_k: 20
+            top_p: 0.75
+            temperature: 0.3
+        output_file: registry/data/arena/answers/answer_Baichuan2-7B-Base.jsonl
+    Qwen-7B:
+        model_id_or_path: qwen/Qwen-7B
+        revision: v1.1.8       # revision of model, default is NULL
+        precision: torch.float16
+        enable: true           # enable or disable this model      # TODO: tokenizer issue
+        template_type: default-generation
+        generation_config:
+            do_sample: true
+            max_new_tokens: 256
+            top_k: 20
+            top_p: 0.75
+            temperature: 0.3
+        output_file: registry/data/arena/answers/answer_Qwen-7B.jsonl
+
+# model of auto-reviewer
+reviews_gen:
+    enable: true
+    reviewer:
+        ref: evalscope.evaluator.reviewer.auto_reviewer:AutoReviewerGpt4
+        args:
+            model: gpt-4
+            max_tokens: 1024
+            temperature: 0
+            # pairwise comparison against baseline
+            mode: pairwise_baseline
+            # position bias mitigation strategy, options: swap_position, randomize_order, None. default is None
+            position_bias_mitigation: swap_position
+            # completion parser config, default is lmsys_parser
+            fn_completion_parser: lmsys_parser
+    # target answers list to be reviewed, could be replaced by your own path: /path/to/answers.jsonl
+    target_answers: [registry/data/arena/answers/answer_chatglm3-6b.jsonl,
+                     registry/data/arena/answers/answer_Baichuan2-7B-Base.jsonl]
+    # the path to the outputs of the baseline model
+    baseline_file: registry/data/arena/answers/answer_text_davinci_003.jsonl
+    # the path to the reference answers
+    reference_file:
+    # prompt templates for auto reviewer(GPT-4)
+    prompt_file: registry/data/prompt_template/lmsys_v2.jsonl
+    # output file of auto reviewer
+    review_file: registry/data/arena/reviews/review_gpt4_pair_baseline.jsonl
+    # cache file of auto reviewer
+    cache_file: registry/data/arena/reviews/review_gpt4_pair_baseline.jsonl
+
+# rating results
+rating_gen:
+    enable: true
+    metrics: ['pairwise']
+    baseline_model: text_davinci_003
+    # elo rating report file
+    report_file: registry/data/arena/reports/rating_pairwise_baseline.csv

evalscope/registry/config/cfg_single.yaml

@@ -0,0 +1,78 @@
+# input raw data
+question_file: registry/data/question.jsonl
+
+# candidate models to be battled
+answers_gen:
+    chatglm3-6b:
+        # model_id_or_path could be local absolute path, e.g. /to/path/.cache/modelscope/ZhipuAI/chatglm3-6b
+        model_id_or_path: ZhipuAI/chatglm3-6b       # model_id on modelscope
+        revision: v1.0.2        # revision of model, default is NULL
+        precision: torch.float16
+        enable: true            # enable or disable this model
+        template_type: chatglm3
+        generation_config:
+            do_sample: true
+            max_new_tokens: 256
+            top_k: 20
+            top_p: 0.75
+            temperature: 0.3
+        # output predicted answer file name
+        output_file: registry/data/arena/answers/answer_chatglm3-6b.jsonl
+    Baichuan2-7B-Base:
+        model_id_or_path: baichuan-inc/Baichuan2-7B-Base
+        revision: v1.0.2       # revision of model, default is NULL
+        precision: torch.float16
+        enable: false           # enable or disable this model
+        template_type: default-generation
+        generation_config:
+            do_sample: true
+            max_new_tokens: 256
+            top_k: 20
+            top_p: 0.75
+            temperature: 0.3
+        output_file: registry/data/arena/answers/answer_Baichuan2-7B-Base.jsonl
+    Qwen-7B:
+        model_id_or_path: qwen/Qwen-7B
+        revision: v1.1.8       # revision of model, default is NULL
+        precision: torch.float16
+        enable: true           # enable or disable this model      # TODO: tokenizer issue
+        template_type: default-generation
+        generation_config:
+            do_sample: true
+            max_new_tokens: 256
+            top_k: 20
+            top_p: 0.75
+            temperature: 0.3
+        output_file: registry/data/arena/answers/answer_Qwen-7B.jsonl
+
+# model of auto-reviewer
+reviews_gen:
+    enable: true
+    reviewer:
+        ref: evalscope.evaluator.reviewer.auto_reviewer:AutoReviewerGpt4
+        args:
+            model: gpt-4
+            max_tokens: 1024
+            temperature: 0
+            # pairwise comparison against baseline
+            mode: single
+            # completion parser config, default is lmsys_parser
+            fn_completion_parser: lmsys_parser
+    # target answers list to be reviewed, could be replaced by your own path: /path/to/answers.jsonl
+    target_answers: [registry/data/arena/answers/answer_chatglm3-6b.jsonl,
+                     registry/data/arena/answers/answer_Baichuan2-7B-Base.jsonl]
+    # the path to the reference answers
+    reference_file:
+    # prompt templates for auto reviewer(GPT-4)
+    prompt_file: registry/data/prompt_template/lmsys_v2.jsonl
+    # output file of auto reviewer
+    review_file: registry/data/arena/reviews/review_gpt4_single.jsonl
+    # cache file of auto reviewer
+    cache_file: registry/data/arena/reviews/review_gpt4_single.jsonl
+
+# rating results
+rating_gen:
+    enable: true
+    metrics: ['score']
+    # elo rating report file
+    report_file: registry/data/arena/reports/rating_single.csv