evalscope 0.13.2__py3-none-any.whl → 0.15.0__py3-none-any.whl

evalscope/perf/utils/benchmark_util.py

@@ -11,7 +11,7 @@ logger = get_logger()
 @dataclass
 class BenchmarkData:
     request: Any = None
-    start_time: float = field(default_factory=time.perf_counter)
+    start_time: float = 0.0
     completed_time: float = 0.0
     chunk_times: List[float] = field(default_factory=list)
     success: bool = False
@@ -32,13 +32,13 @@ class BenchmarkData:
         self.query_latency = self.completed_time - self.start_time
         if len(self.chunk_times) > 1:
             self.first_chunk_latency = self.chunk_times[0] - self.start_time
-            self.n_chunks = len(self.chunk_times) - 2
+            self.n_chunks = len(self.chunk_times) - 2  # remove last and first chunk
             self.n_chunks_time = self.chunk_times[-2] - self.chunk_times[0]
         else:
             self.first_chunk_latency = self.query_latency
             self.n_chunks = 1
             self.n_chunks_time = self.query_latency
-        self.time_per_output_token = self.query_latency / self.completion_tokens
+        self.time_per_output_token = self.n_chunks_time / self.completion_tokens
 
     def _calculate_tokens(self, api_plugin):
         self.prompt_tokens, self.completion_tokens = \
@@ -73,7 +73,9 @@ class BenchmarkMetrics:
     avg_chunk_time: float = -1
     avg_prompt_tokens: float = -1
     avg_completion_tokens: float = -1
-    avg_token_per_seconds: float = -1
+    avg_input_token_per_seconds: float = -1
+    avg_output_token_per_seconds: float = -1
+    avg_total_token_per_seconds: float = -1
     avg_time_per_token: float = -1
     qps: float = -1
 
@@ -111,22 +113,26 @@ class BenchmarkMetrics:
             self.avg_chunk_time = self.total_chunks_time / self.n_total_chunks
             self.avg_prompt_tokens = self.n_total_prompt_tokens / self.n_succeed_queries
             self.avg_completion_tokens = self.n_total_completion_tokens / self.n_succeed_queries
-            self.avg_token_per_seconds = self.n_total_completion_tokens / self.total_time
+            self.avg_input_token_per_seconds = self.n_total_prompt_tokens / self.total_first_chunk_latency
+            self.avg_output_token_per_seconds = self.n_total_completion_tokens / self.total_time
+            self.avg_total_token_per_seconds = (self.n_total_prompt_tokens
+                                                + self.n_total_completion_tokens) / self.total_time
             self.avg_time_per_token = self.n_time_per_output_token / self.n_succeed_queries
             self.qps = self.n_succeed_queries / self.total_time
         except ZeroDivisionError as e:
             logger.exception(e)
             return
 
-    def create_message(self, default_ndigits=3):
+    def create_message(self, default_ndigits=4):
         message = {
             'Time taken for tests (s)': round(self.total_time, default_ndigits),
             'Number of concurrency': self.concurrency,
             'Total requests': int(self.n_total_queries),
             'Succeed requests': self.n_succeed_queries,
             'Failed requests': self.n_failed_queries,
-            'Throughput(average tokens/s)': round(self.avg_token_per_seconds, default_ndigits),
-            'Average QPS': round(self.qps, default_ndigits),
+            'Output token throughput (tok/s)': round(self.avg_output_token_per_seconds, default_ndigits),
+            'Total token throughput (tok/s)': round(self.avg_total_token_per_seconds, default_ndigits),
+            'Request throughput (req/s)': round(self.qps, default_ndigits),
             'Average latency (s)': round(self.avg_latency, default_ndigits),
             'Average time to first token (s)': round(self.avg_first_chunk_latency, default_ndigits),
             'Average time per output token (s)': round(self.avg_time_per_token, default_ndigits),

evalscope/perf/utils/db_util.py

@@ -165,6 +165,7 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
     CHUNK_TIMES_INDEX = 1
     LATENCY_INDEX = 4
     FIRST_CHUNK_LATENCY_INDEX = 5
+    CHUNK_TIME_INDEX = 7
     PROMPT_TOKENS_INDEX = 8
     COMPLETION_TOKENS_INDEX = 9
 
@@ -175,14 +176,19 @@ def get_percentile_results(result_db_path: str) -> Dict[str, List[float]]:
 
     metrics = {
         'TTFT (s)': [row[FIRST_CHUNK_LATENCY_INDEX] for row in rows],
-        'TPOT (s)':
+        'ITL (s)':
         inter_token_latencies_all,
+        'TPOT (s)':
+        [(row[CHUNK_TIME_INDEX] / row[COMPLETION_TOKENS_INDEX]) if row[COMPLETION_TOKENS_INDEX] > 0 else float('nan')
+         for row in rows],
         'Latency (s)': [row[LATENCY_INDEX] for row in rows],
         'Input tokens': [row[PROMPT_TOKENS_INDEX] for row in rows],
         'Output tokens': [row[COMPLETION_TOKENS_INDEX] for row in rows],
-        'Throughput(tokens/s)':
+        'Output throughput(tok/s)':
         [(row[COMPLETION_TOKENS_INDEX] / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan')
-         for row in rows]
+         for row in rows],
+        'Total throughput(tok/s)': [((row[PROMPT_TOKENS_INDEX] + row[COMPLETION_TOKENS_INDEX])
+                                     / row[LATENCY_INDEX]) if row[LATENCY_INDEX] > 0 else float('nan') for row in rows]
     }
 
     # Calculate percentiles for each metric

evalscope/perf/utils/log_utils.py

@@ -0,0 +1,41 @@
+import os
+
+from evalscope.perf.arguments import Arguments
+
+
+def init_wandb(args: Arguments) -> None:
+    """
+    Initialize WandB for logging.
+    """
+    # Initialize wandb if the api key is provided
+    import datetime
+    try:
+        import wandb
+    except ImportError:
+        raise RuntimeError('Cannot import wandb. Please install it with command: \n pip install wandb')
+    os.environ['WANDB_SILENT'] = 'true'
+    os.environ['WANDB_DIR'] = args.outputs_dir
+
+    wandb.login(key=args.wandb_api_key)
+    current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    name = args.name if args.name else f'{args.model_id}_{current_time}'
+    wandb.init(project='perf_benchmark', name=name, config=args.to_dict())
+
+
+def init_swanlab(args: Arguments) -> None:
+    import datetime
+    try:
+        import swanlab
+    except ImportError:
+        raise RuntimeError('Cannot import swanlab. Please install it with command: \n pip install swanlab')
+    os.environ['SWANLAB_LOG_DIR'] = args.outputs_dir
+    if not args.swanlab_api_key == 'local':
+        swanlab.login(api_key=args.swanlab_api_key)
+    current_time = datetime.datetime.now().strftime('%Y%m%d_%H%M%S')
+    name = args.name if args.name else f'{args.model_id}_{current_time}'
+    swanlab.config.update({'framework': '📏evalscope'})
+    swanlab.init(
+        project='perf_benchmark',
+        name=name,
+        config=args.to_dict(),
+        mode='local' if args.swanlab_api_key == 'local' else None)

evalscope/report/__init__.py

@@ -1,5 +1,6 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+from evalscope.report.app_arguments import add_argument
 from evalscope.report.combinator import gen_table, get_data_frame, get_report_list
 from evalscope.report.generator import ReportGenerator
 from evalscope.report.utils import Category, Report, ReportKey, Subset

evalscope/report/app.py

@@ -11,7 +11,7 @@ from dataclasses import dataclass
 from typing import Any, List, Union
 
 from evalscope.constants import DataCollection
-from evalscope.report import Report, ReportKey, get_data_frame, get_report_list
+from evalscope.report import Report, ReportKey, add_argument, get_data_frame, get_report_list
 from evalscope.utils.io_utils import OutputsStructure, yaml_to_dict
 from evalscope.utils.logger import configure_logging, get_logger
 from evalscope.version import __version__
@@ -22,6 +22,23 @@ PLOTLY_THEME = 'plotly_dark'
 REPORT_TOKEN = '@@'
 MODEL_TOKEN = '::'
 DATASET_TOKEN = ', '
+LATEX_DELIMITERS = [{
+    'left': '$$',
+    'right': '$$',
+    'display': True
+}, {
+    'left': '$',
+    'right': '$',
+    'display': False
+}, {
+    'left': '\\(',
+    'right': '\\)',
+    'display': False
+}, {
+    'left': '\\[',
+    'right': '\\]',
+    'display': True
+}]
 
 
 def scan_for_report_folders(root_path):
@@ -44,7 +61,7 @@ def scan_for_report_folders(root_path):
                 continue
             datasets = []
             for dataset_item in glob.glob(os.path.join(model_item, '*.json')):
-                datasets.append(os.path.basename(dataset_item).split('.')[0])
+                datasets.append(os.path.splitext(os.path.basename(dataset_item))[0])
             datasets = DATASET_TOKEN.join(datasets)
             reports.append(
                 f'{os.path.basename(folder)}{REPORT_TOKEN}{os.path.basename(model_item)}{MODEL_TOKEN}{datasets}')
@@ -234,6 +251,18 @@ def convert_html_tags(text):
     return text
 
 
+def convert_markdown_image(text):
+    if not os.path.isfile(text):
+        return text
+    # Convert the image path to a markdown image tag
+    if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
+        text = os.path.abspath(text)
+        image_tag = f'![image](gradio_api/file={text})'
+        logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+        return image_tag
+    return text
+
+
 def process_string(string: str, max_length: int = 2048) -> str:
     string = convert_html_tags(string)  # for display labels e.g. `<think>`
     if len(string) > max_length:
@@ -253,17 +282,17 @@ def process_model_prediction(item: Any):
 
 
 def normalize_score(score):
-    if isinstance(score, bool):
-        return 1.0 if score else 0.0
-    elif isinstance(score, dict):
-        for key in score:
-            return float(score[key])
-        return 0.0
-    else:
-        try:
-            return float(score)
-        except (ValueError, TypeError):
+    try:
+        if isinstance(score, bool):
+            return 1.0 if score else 0.0
+        elif isinstance(score, dict):
+            for key in score:
+                return float(score[key])
             return 0.0
+        else:
+            return float(score)
+    except (ValueError, TypeError):
+        return 0.0
 
 
 def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subset_name: str):
@@ -285,7 +314,7 @@ def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subs
                 'Input': raw_input,
                 'Generated': raw_pred_answer,
                 'Gold': parsed_gold_answer if parsed_gold_answer != raw_input else '*Same as Input*',
-                'Pred': parsed_pred_answer if parsed_pred_answer != raw_pred_answer else '*Same as Generated*',
+                'Pred': parsed_pred_answer,
                 'Score': score,
                 'NScore': normalize_score(score)
             }
@@ -295,22 +324,6 @@ def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subs
     return df_subset
 
 
-def get_table_data(data_review_df: pd.DataFrame, page: int = 1, rows_per_page: int = 1) -> pd.DataFrame:
-    if data_review_df is None:
-        return pd.DataFrame(), None
-
-    logger.debug(f'page: {page}, rows_per_page: {rows_per_page}')
-    start = (page - 1) * rows_per_page
-    end = start + rows_per_page
-    df_subset = data_review_df.iloc[start:end].copy()
-    df_subset['Input'] = df_subset['Input'].map(process_model_prediction).astype(str)
-    df_subset['Generated'] = df_subset['Generated'].map(process_model_prediction).astype(str)
-    df_subset['Pred'] = df_subset['Pred'].map(process_model_prediction).astype(str)
-    df_subset['Score'] = df_subset['Score'].map(process_model_prediction).astype(str)
-    styler = style_df(df_subset, columns=['NScore'])
-    return df_subset, styler
-
-
 @dataclass
 class SidebarComponents:
     root_path: gr.Textbox
@@ -457,7 +470,11 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
         'page': {
             'zh': '页码',
             'en': 'Page'
-        }
+        },
+        'score_threshold': {
+            'zh': '分数阈值',
+            'en': 'Score Threshold'
+        },
     }
 
     # Update the UI components with localized labels
@@ -489,37 +506,53 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
         gr.Markdown(f'### {locale_dict["model_prediction"][lang]}')
         subset_select = gr.Dropdown(
             label=locale_dict['select_subset'][lang], choices=[], show_label=True, interactive=True)
+
         with gr.Row():
             answer_mode_radio = gr.Radio(
                 label=locale_dict['answer_mode'][lang], choices=['All', 'Pass', 'Fail'], value='All', interactive=True)
-            page_number = gr.Number(
-                value=1, label=locale_dict['page'][lang], minimum=1, maximum=1, step=1, interactive=True)
-        answer_mode_counts = gr.Markdown('', label='Counts')
+            score_threshold = gr.Number(value=0.99, label=locale_dict['score_threshold'][lang], interactive=True)
+
         data_review_df = gr.State(None)
         filtered_review_df = gr.State(None)
-        data_review_table = gr.DataFrame(
-            value=None,
-            datatype=['markdown', 'markdown', 'markdown', 'markdown', 'markdown', 'number'],
-            # column_widths=['500px', '500px'],
-            wrap=True,
-            latex_delimiters=[{
-                'left': '$$',
-                'right': '$$',
-                'display': True
-            }, {
-                'left': '$',
-                'right': '$',
-                'display': False
-            }, {
-                'left': '\\(',
-                'right': '\\)',
-                'display': False
-            }, {
-                'left': '\\[',
-                'right': '\\]',
-                'display': True
-            }],
-            max_height=600)
+
+        # show statistics
+        with gr.Row(variant='panel'):
+            with gr.Column():
+                gr.Markdown('### *Counts*')
+                answer_mode_counts = gr.Markdown('')
+            with gr.Column():
+                page_number = gr.Number(
+                    value=1, label=locale_dict['page'][lang], minimum=1, maximum=1, step=1, interactive=True)
+
+        # show data review table
+        with gr.Row(variant='panel'):
+            with gr.Column():
+                gr.Markdown('### *Score*')
+                score_text = gr.Markdown(
+                    '', elem_id='score_text', latex_delimiters=LATEX_DELIMITERS, show_copy_button=True)
+            with gr.Column():
+                gr.Markdown('### *Normalized Score*')
+                nscore = gr.Markdown('', elem_id='score_text', latex_delimiters=LATEX_DELIMITERS)
+
+        with gr.Row(variant='panel'):
+            with gr.Column():
+                gr.Markdown('### *Gold*')
+                gold_text = gr.Markdown(
+                    '', elem_id='gold_text', latex_delimiters=LATEX_DELIMITERS, show_copy_button=True)
+            with gr.Column():
+                gr.Markdown('### *Pred*')
+                pred_text = gr.Markdown(
+                    '', elem_id='pred_text', latex_delimiters=LATEX_DELIMITERS, show_copy_button=True)
+
+        with gr.Row(variant='panel'):
+            with gr.Column():
+                gr.Markdown('### *Input*')
+                input_text = gr.Markdown(
+                    '', elem_id='input_text', latex_delimiters=LATEX_DELIMITERS, show_copy_button=True)
+            with gr.Column():
+                gr.Markdown('### *Generated*')
+                generated_text = gr.Markdown(
+                    '', elem_id='generated_text', latex_delimiters=LATEX_DELIMITERS, show_copy_button=True)
 
     @report_name.change(
         inputs=[sidebar.root_path, report_name],
@@ -561,15 +594,15 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
         return data_review_df, 1
 
     @gr.on(
-        triggers=[data_review_df.change, answer_mode_radio.change],
-        inputs=[data_review_df, answer_mode_radio],
+        triggers=[data_review_df.change, answer_mode_radio.change, score_threshold.change],
+        inputs=[data_review_df, answer_mode_radio, score_threshold],
         outputs=[filtered_review_df, page_number, answer_mode_counts])
-    def filter_data(data_review_df, answer_mode):
+    def filter_data(data_review_df, answer_mode, score_threshold):
         if data_review_df is None:
             return None, gr.update(value=1, maximum=1), ''
 
         all_count = len(data_review_df)
-        pass_df = data_review_df[data_review_df['NScore'] >= 0.99]
+        pass_df = data_review_df[data_review_df['NScore'] >= score_threshold]
         pass_count = len(pass_df)
         fail_count = all_count - pass_count
 
@@ -578,7 +611,7 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
         if answer_mode == 'Pass':
             filtered_df = pass_df
         elif answer_mode == 'Fail':
-            filtered_df = data_review_df[data_review_df['NScore'] < 0.99]
+            filtered_df = data_review_df[data_review_df['NScore'] < score_threshold]
         else:
             filtered_df = data_review_df
 
@@ -588,13 +621,33 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
 
     @gr.on(
         triggers=[filtered_review_df.change, page_number.change],
-        inputs=[filtered_review_df, page_number],
-        outputs=[data_review_table])
-    def update_table(filtered_df, page_number):
-        if filtered_df is None:
-            return gr.update(value=None)
-        subset_df, styler = get_table_data(filtered_df, page_number)
-        return styler
+        inputs=[filtered_review_df, page_number, score_threshold],
+        outputs=[input_text, generated_text, gold_text, pred_text, score_text, nscore])
+    def update_table_components(filtered_df, page_number, score_threshold):
+        if filtered_df is None or len(filtered_df) == 0:
+            return '', '', '', '', '', ''
+
+        # Get single row data for the current page
+        start = (page_number - 1)
+        if start >= len(filtered_df):
+            return '', '', '', '', '', ''
+
+        row = filtered_df.iloc[start]
+
+        # Process the data for display
+        input_md = process_model_prediction(row['Input'])
+        generated_md = process_model_prediction(row['Generated'])
+        gold_md = process_model_prediction(row['Gold'])
+        pred_md = convert_markdown_image(process_model_prediction(row['Pred']))
+        score_md = process_model_prediction(row['Score'])
+        nscore_val = float(row['NScore']) if not pd.isna(row['NScore']) else 0.0
+
+        if nscore_val >= score_threshold:
+            nscore_val = f'<div style="background-color:rgb(45,104, 62); padding:10px;">{nscore_val}</div>'
+        else:
+            nscore_val = f'<div style="background-color:rgb(151, 31, 44); padding:10px;">{nscore_val}</div>'
+
+        return input_md, generated_md, gold_md, pred_md, score_md, nscore_val
 
     return SingleModelComponents(report_name=report_name)
 
@@ -696,16 +749,13 @@ def create_app(args: argparse.Namespace):
             text = '<' if new_visible else '>'
             return gr.update(visible=new_visible), new_visible, gr.update(value=text)
 
-    demo.launch(share=args.share, server_name=args.server_name, server_port=args.server_port, debug=args.debug)
-
-
-def add_argument(parser: argparse.ArgumentParser):
-    parser.add_argument('--share', action='store_true', help='Share the app.')
-    parser.add_argument('--server-name', type=str, default='0.0.0.0', help='The server name.')
-    parser.add_argument('--server-port', type=int, default=None, help='The server port.')
-    parser.add_argument('--debug', action='store_true', help='Debug the app.')
-    parser.add_argument('--lang', type=str, default='zh', help='The locale.', choices=['zh', 'en'])
-    parser.add_argument('--outputs', type=str, default='./outputs', help='The outputs dir.')
+    demo.launch(
+        share=args.share,
+        server_name=args.server_name,
+        server_port=args.server_port,
+        debug=args.debug,
+        allowed_paths=args.allowed_paths,
+    )
 
 
 if __name__ == '__main__':

evalscope/report/app_arguments.py

@@ -0,0 +1,11 @@
+import argparse
+
+
+def add_argument(parser: argparse.ArgumentParser):
+    parser.add_argument('--share', action='store_true', help='Share the app.')
+    parser.add_argument('--server-name', type=str, default='0.0.0.0', help='The server name.')
+    parser.add_argument('--server-port', type=int, default=None, help='The server port.')
+    parser.add_argument('--debug', action='store_true', help='Debug the app.')
+    parser.add_argument('--lang', type=str, default='zh', help='The locale.', choices=['zh', 'en'])
+    parser.add_argument('--outputs', type=str, default='./outputs', help='The outputs dir.')
+    parser.add_argument('--allowed-paths', nargs='+', default=['/'], help='The outputs dir.')

evalscope/report/generator.py

@@ -48,7 +48,7 @@ class ReportGenerator:
         df = flatten_subset()
 
         metrics_list = []
-        for metric_name, group_metric in df.groupby('metric_name'):
+        for metric_name, group_metric in df.groupby('metric_name', sort=False):
             categories = []
             for category_name, group_category in group_metric.groupby('categories'):
                 subsets = []

evalscope/run.py

@@ -58,10 +58,17 @@ def setup_work_directory(task_cfg: TaskConfig, run_time: str):
 
     outputs = OutputsStructure(outputs_dir=task_cfg.work_dir)
 
+    # Unify the output directory structure
     if task_cfg.eval_backend == EvalBackend.OPEN_COMPASS:
         task_cfg.eval_config['time_str'] = run_time
     elif task_cfg.eval_backend == EvalBackend.VLM_EVAL_KIT:
         task_cfg.eval_config['work_dir'] = task_cfg.work_dir
+    elif task_cfg.eval_backend == EvalBackend.RAG_EVAL:
+        from evalscope.backend.rag_eval import Tools
+        if task_cfg.eval_config['tool'].lower() == Tools.MTEB:
+            task_cfg.eval_config['eval']['output_folder'] = task_cfg.work_dir
+        elif task_cfg.eval_config['tool'].lower() == Tools.CLIP_BENCHMARK:
+            task_cfg.eval_config['eval']['output_dir'] = task_cfg.work_dir
     return outputs
 
 
@@ -146,10 +153,10 @@ def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsSt
         data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
         return EvaluatorCollection(task_cfg, data_adapter, outputs, base_model)
 
-    # Initialize model adapter
-    model_adapter = initialize_model_adapter(task_cfg, benchmark, base_model)
-    # Initialize data adapter
+    # Initialize data adapter first to update config
     data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
+    # Initialize model adapter
+    model_adapter = initialize_model_adapter(task_cfg, data_adapter, base_model)
 
     # update task_cfg.dataset_args
     task_cfg.dataset_args[dataset_name] = benchmark.to_string_dict()

evalscope/summarizer.py

@@ -105,7 +105,8 @@ class Summarizer:
                             summary_res: dict = csv_to_list(summary_file_path)[0]
                         elif summary_file_path.endswith('json'):
                             summary_res: dict = json_to_dict(summary_file_path)
-                        file_name = os.path.basename(summary_file_path).split('.')[0]
+                        base_name = os.path.basename(summary_file_path)
+                        file_name = os.path.splitext(base_name)[0]
                         final_res_list.append({file_name: summary_res})
 
             elif eval_backend == EvalBackend.THIRD_PARTY:

evalscope/third_party/thinkbench/eval.py

@@ -357,7 +357,7 @@ judge_config = dict(
 )
 
 distill_qwen_config = dict(
-    report_path = './outputs/20250218_180219',
+    report_path = '../eval-scope/outputs/20250218_180219',
     model_name = 'DeepSeek-R1-Distill-Qwen-7B',
     tokenizer_path = 'deepseek-ai/DeepSeek-R1-Distill-Qwen-7B',
     dataset_name = 'math_500',
@@ -367,7 +367,7 @@ distill_qwen_config = dict(
 )
 
 math_qwen_config = dict(
-    report_path = './outputs/20250219_202358',
+    report_path = '../eval-scope/outputs/20250219_202358',
     model_name = 'Qwen2.5-Math-7B-Instruct',
     tokenizer_path = 'Qwen/Qwen2.5-Math-7B-Instruct',
     dataset_name = 'math_500',
@@ -377,7 +377,7 @@ math_qwen_config = dict(
 )
 
 r1_config = dict(
-    report_path = './outputs/20250307_000404',
+    report_path = '../eval-scope/outputs/20250307_000404',
     model_name = 'deepseek-r1',
     tokenizer_path = 'deepseek-ai/DeepSeek-R1',
     dataset_name = 'math_500',
@@ -387,7 +387,7 @@ r1_config = dict(
 )
 
 qwq_preview_config = dict(
-    report_path = './outputs/20250221_105911',
+    report_path = '../eval-scope/outputs/20250221_105911',
     model_name = 'qwq-32b-preview',
     tokenizer_path = 'Qwen/QwQ-32B-Preview',
     dataset_name = 'math_500',
@@ -397,7 +397,7 @@ qwq_preview_config = dict(
 )
 
 qwq_config = dict(
-    report_path = './outputs/20250306_181550',
+    report_path = '../eval-scope/outputs/20250306_181550',
     model_name = 'QwQ-32B',
     tokenizer_path = 'Qwen/QwQ-32B',
     dataset_name = 'math_500',
@@ -407,7 +407,7 @@ qwq_config = dict(
 )
 
 distill_qwen_32b = dict(
-    report_path = './outputs/20250306_235951',
+    report_path = '../eval-scope/outputs/20250306_235951',
     model_name = 'deepseek-r1-distill-qwen-32b',
     tokenizer_path = 'deepseek-ai/DeepSeek-R1-Distill-Qwen-32B',
     dataset_name = 'math_500',
@@ -416,14 +416,26 @@ distill_qwen_32b = dict(
     judge_config=judge_config
 )
 
+qwen3_32b_think = dict(
+    report_path = '../eval-scope/outputs/20250428_151817',
+    model_name = 'Qwen3-32B',
+    tokenizer_path = 'Qwen/Qwen3-32B',
+    dataset_name = 'math_500',
+    subsets = ['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
+    split_strategies='separator',
+    judge_config=judge_config
+)
+
 if __name__ == '__main__':
     # run_task(distill_qwen_config, count=80)
     # run_task(math_qwen_config)
     # run_task(qwq_preview_config, max_tokens=20000, count=200, workers=128)
     # run_task(r1_config, max_tokens=20000, count=200, workers=128)
     # run_task(qwq_config, max_tokens=20000, count=200, workers=128)
+    run_task(qwen3_32b_think, max_tokens=20000, count=200, workers=128)
     # run_task(distill_qwen_32b, max_tokens=20000, count=200, workers=128)
 
     # combine_results([qwq_config, r1_config, qwq_preview_config,  distill_qwen_32b], output_path='outputs/model_comparison_metrics.png')
     # combine_results([qwq_config, r1_config, distill_qwen_32b], output_path='outputs/model_comparison_metrics_3models.png')
-    combine_results([distill_qwen_config, math_qwen_config, qwq_config, r1_config, qwq_preview_config, distill_qwen_32b], output_path='outputs/model_comparison_metrics_6models.png')
+    # combine_results([distill_qwen_config, math_qwen_config, qwq_config, r1_config, qwq_preview_config, distill_qwen_32b], output_path='outputs/model_comparison_metrics_6models.png')
+    combine_results([qwq_config, r1_config, distill_qwen_32b, qwen3_32b_think], output_path='outputs/model_comparison_metrics_4models.png')

evalscope/utils/chat_service.py

@@ -64,10 +64,10 @@ class ChatCompletionResponseStreamChoice(BaseModel):
 
 class ChatCompletionResponse(BaseModel):
     model: str
-    object: Literal['chat.completion', 'chat.completion.chunk']
+    object: Literal['chat.completion', 'chat.completion.chunk', 'images.generations']
     choices: List[Union[ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, Any]]
     created: Optional[int] = Field(default_factory=lambda: int(time.time()))
-    usage: Optional[Usage]
+    usage: Optional[Usage] = None
 
 
 class TextCompletionRequest(BaseModel):